From 94c3b93ff98fb3881c4f79fecf697425ad5c3fff Mon Sep 17 00:00:00 2001
From: MrCroxx <mrcroxx@outlook.com>
Date: Thu, 18 May 2023 14:36:38 +0800
Subject: [PATCH 001/261] feat: introduce intrusive double link list

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock                        |  17 +
 Cargo.toml                        |   2 +
 src/collections/dlist.rs          | 551 ++++++++++++++++++++++++++++++
 src/{utils => collections}/mod.rs |   3 +-
 src/lib.rs                        |   2 +-
 src/utils/count_min_sketch.rs     | 357 -------------------
 src/utils/hash.rs                 |  52 ---
 7 files changed, 572 insertions(+), 412 deletions(-)
 create mode 100644 src/collections/dlist.rs
 rename src/{utils => collections}/mod.rs (93%)
 delete mode 100644 src/utils/count_min_sketch.rs
 delete mode 100644 src/utils/hash.rs

diff --git a/Cargo.lock b/Cargo.lock
index afea6a3a..5e4c9e08 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -42,6 +42,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "either"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+
 [[package]]
 name = "foyer"
 version = "0.1.0"
@@ -49,6 +55,8 @@ dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
  "futures",
+ "itertools",
+ "memoffset",
  "parking_lot",
  "paste",
  "rand_mt",
@@ -143,6 +151,15 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.144"
diff --git a/Cargo.toml b/Cargo.toml
index af4da56c..24f6fcff 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,8 @@ license = "Apache-2.0"
 [dependencies]
 crossbeam-epoch = "0.9"
 futures = "0.3"
+itertools = "0.10.5"
+memoffset = "0.8"
 parking_lot = "0.12"
 paste = "1.0"
 
diff --git a/src/collections/dlist.rs b/src/collections/dlist.rs
new file mode 100644
index 00000000..30d57d03
--- /dev/null
+++ b/src/collections/dlist.rs
@@ -0,0 +1,551 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::marker::PhantomData;
+use std::ptr::NonNull;
+
+/// An entry in an intrusive double linked list
+#[derive(Default, Debug)]
+pub struct Entry {
+    /// The prev entry ptr in the double linked list.
+    prev: Option<NonNull<Entry>>,
+
+    /// The next entry ptr in the double linked list.
+    next: Option<NonNull<Entry>>,
+}
+
+pub trait Adapter<E> {
+    /// entry ptr to element ptr
+    fn en2el(_: NonNull<Entry>) -> NonNull<E>;
+
+    /// element ptr to entry ptr
+    fn el2en(_: NonNull<E>) -> NonNull<Entry>;
+}
+
+#[macro_export]
+macro_rules! intrusive_dlist {
+    (
+        $element:ident $(< $( $lt:tt $( : $clt:tt $(+ $dlt:tt)* )? ),+ >)?, $entry:ident, $adapter:ident
+    ) => {
+        struct $adapter;
+
+        impl $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? $crate::collections::dlist::Adapter<$element $(< $( $lt ),+ >)?> for $adapter  {
+            fn en2el(entry: NonNull<Entry>) -> NonNull<$element $(< $( $lt ),+ >)?> {
+                unsafe {
+                    let ptr = (entry.as_ptr() as *mut u8)
+                        .sub(memoffset::offset_of!($element $(< $( $lt ),+ >)?, $entry))
+                        .cast::<$element $(< $( $lt ),+ >)?>();
+                    NonNull::new_unchecked(ptr)
+                }
+            }
+
+            fn el2en(element: NonNull<$element $(< $( $lt ),+ >)?>) -> NonNull<Entry> {
+                unsafe {
+                    let ptr = (element.as_ptr() as *mut u8)
+                        .add(memoffset::offset_of!($element $(< $( $lt ),+ >)?, $entry))
+                        .cast::<Entry>();
+                    NonNull::new_unchecked(ptr)
+                }
+            }
+        }
+    };
+}
+
+/// TODO: write docs
+#[derive(Debug)]
+pub struct DList<E, A: Adapter<E>> {
+    /// head ptr of the double linked list
+    head: Option<NonNull<Entry>>,
+
+    /// tail ptr of the double linked list
+    tail: Option<NonNull<Entry>>,
+
+    /// length of the double linked list
+    len: usize,
+
+    _marker: PhantomData<(E, A)>,
+}
+
+impl<E, A: Adapter<E>> Default for DList<E, A> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<E, A: Adapter<E>> DList<E, A> {
+    /// create an empty intrusive double linklist
+    pub fn new() -> Self {
+        Self {
+            head: None,
+            tail: None,
+            len: 0,
+            _marker: PhantomData::default(),
+        }
+    }
+
+    /// link a dangling element to the list at head
+    ///
+    /// # Safety
+    ///
+    /// `element` must not be in the list
+    pub unsafe fn link_at_head(&mut self, element: NonNull<E>) {
+        let mut entry = A::el2en(element);
+        assert_ne!(Some(entry), self.head);
+
+        entry.as_mut().next = self.head;
+        entry.as_mut().prev = None;
+
+        // fix the prev ptr of head
+        if let Some(head) = &mut self.head {
+            head.as_mut().prev = Some(entry);
+        }
+        self.head = Some(entry);
+
+        if self.tail.is_none() {
+            self.tail = Some(entry)
+        }
+
+        self.len += 1;
+    }
+
+    /// link a dangling element to the list at tail
+    ///
+    /// # Safety
+    ///
+    /// `element` must not be in the list
+    pub unsafe fn link_at_tail(&mut self, element: NonNull<E>) {
+        let mut entry = A::el2en(element);
+        assert_ne!(Some(entry), self.tail);
+
+        entry.as_mut().next = None;
+        entry.as_mut().prev = self.tail;
+
+        // fix the prev ptr of tail
+        if let Some(tail) = &mut self.tail {
+            tail.as_mut().next = Some(entry);
+        }
+        self.tail = Some(entry);
+
+        if self.head.is_none() {
+            self.head = Some(entry)
+        }
+
+        self.len += 1;
+    }
+
+    /// link a dangling element to the list before `next_element`
+    ///
+    /// # Safety
+    ///
+    /// `element` must not be in the list
+    /// `next_element` must be in the list
+    pub unsafe fn link_before(&mut self, next_element: NonNull<E>, element: NonNull<E>) {
+        let mut next_entry = A::el2en(next_element);
+        let mut entry = A::el2en(element);
+        assert_ne!(next_entry, entry);
+
+        let mut prev = next_entry.as_mut().prev;
+        assert_ne!(prev, Some(entry));
+
+        entry.as_mut().prev = prev;
+        match &mut prev {
+            Some(prev) => prev.as_mut().next = Some(entry),
+            None => self.head = Some(entry),
+        }
+
+        next_entry.as_mut().prev = Some(entry);
+        entry.as_mut().next = Some(next_entry);
+
+        self.len += 1;
+    }
+
+    /// unlink an element from the list
+    ///
+    /// # Safety
+    ///
+    /// `element` must be in the list
+    pub unsafe fn unlink(&mut self, element: NonNull<E>) {
+        assert!(self.len > 0);
+
+        let mut entry = A::el2en(element);
+
+        // fix head and tail if node is either of that
+        let mut prev = entry.as_mut().prev;
+        let mut next = entry.as_mut().next;
+        if Some(entry) == self.head {
+            self.head = next;
+        }
+        if Some(entry) == self.tail {
+            self.tail = prev;
+        }
+
+        // fix the next and prev ptrs of the node before and after this
+        if let Some(prev) = &mut prev {
+            prev.as_mut().next = next;
+        }
+        if let Some(next) = &mut next {
+            next.as_mut().prev = prev;
+        }
+
+        self.len -= 1;
+    }
+
+    /// remove an element from the list and cleanup its pointers
+    ///
+    /// # Safety
+    ///
+    /// `element` must be in the list
+    pub unsafe fn remove(&mut self, element: NonNull<E>) {
+        self.unlink(element);
+        let mut entry = A::el2en(element);
+        entry.as_mut().next = None;
+        entry.as_mut().prev = None;
+    }
+
+    /// replace an element with an dangling element
+    ///
+    /// # Safety
+    ///
+    /// `old_element` must be in the list
+    /// `new_element` must not be in the list
+    pub unsafe fn replace(&mut self, old_element: NonNull<E>, new_element: NonNull<E>) {
+        let mut old_entry = A::el2en(old_element);
+        let mut new_entry = A::el2en(new_element);
+
+        // update head and tail links if needed
+        if Some(old_entry) == self.head {
+            self.head = Some(new_entry);
+        }
+        if Some(old_entry) == self.tail {
+            self.tail = Some(new_entry);
+        }
+
+        let mut prev = old_entry.as_mut().prev;
+        let mut next = old_entry.as_mut().next;
+
+        // make the prev and next entry point to the new entry
+        if let Some(prev) = &mut prev {
+            prev.as_mut().next = Some(new_entry);
+        }
+        if let Some(next) = &mut next {
+            next.as_mut().prev = Some(new_entry);
+        }
+
+        // make the new entry point to the prev and next entry
+        new_entry.as_mut().prev = prev;
+        new_entry.as_mut().next = next;
+
+        // cleanup the old node
+        old_entry.as_mut().prev = None;
+        old_entry.as_mut().next = None;
+    }
+
+    /// move an element to the head of the list
+    ///
+    /// # Safety
+    ///
+    /// `element` must be in the list
+    pub unsafe fn move_to_head(&mut self, element: NonNull<E>) {
+        let entry = A::el2en(element);
+        if Some(entry) == self.head {
+            return;
+        }
+        self.unlink(element);
+        self.link_at_head(element);
+    }
+
+    /// move an element to the tail of the list
+    ///
+    /// # Safety
+    ///
+    /// `element` must be in the list
+    pub unsafe fn move_to_tail(&mut self, element: NonNull<E>) {
+        let entry = A::el2en(element);
+        if Some(entry) == self.tail {
+            return;
+        }
+        self.unlink(element);
+        self.link_at_tail(element);
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len == 0
+    }
+
+    /// create a iterator of the list from head
+    ///
+    /// # Safety
+    ///
+    /// there is no guarantee that the element cannot be modified while iterating
+    pub unsafe fn iter(&mut self) -> Iter<'_, E, A> {
+        let entry = self.head;
+        Iter { dlist: self, entry }
+    }
+}
+
+pub struct Iter<'a, E, A: Adapter<E>> {
+    dlist: &'a mut DList<E, A>,
+    entry: Option<NonNull<Entry>>,
+}
+
+impl<'a, E, A: Adapter<E>> Iter<'a, E, A> {
+    /// get element ptr
+    pub fn element(&self) -> Option<NonNull<E>> {
+        self.entry.map(|entry| A::en2el(entry))
+    }
+
+    /// move to next
+    ///
+    /// # Safety
+    ///
+    /// the iterator must be in a valid position
+    pub unsafe fn next(&mut self) {
+        self.entry = self.entry.unwrap().as_ref().next
+    }
+
+    /// move to prev
+    ///
+    /// # Safety
+    ///
+    /// the iterator must be in a valid position
+    pub unsafe fn prev(&mut self) {
+        self.entry = self.entry.unwrap().as_ref().prev
+    }
+
+    /// move to head
+    pub fn head(&mut self) {
+        self.entry = self.dlist.head
+    }
+
+    /// move to tail
+    pub fn tail(&mut self) {
+        self.entry = self.dlist.tail
+    }
+}
+
+impl<'a, E, A: Adapter<E>> Iterator for Iter<'a, E, A> {
+    type Item = NonNull<E>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.element() {
+            Some(element) => {
+                unsafe { self.next() };
+                Some(element)
+            }
+            None => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use itertools::Itertools;
+
+    #[derive(Debug)]
+    struct SE {
+        e: Entry,
+        data: usize,
+    }
+
+    impl SE {
+        fn new(data: usize) -> Self {
+            Self {
+                e: Entry::default(),
+                data,
+            }
+        }
+    }
+
+    impl Drop for SE {
+        fn drop(&mut self) {
+            self.data = 0
+        }
+    }
+
+    #[derive(Debug)]
+    struct DE {
+        e1: Entry,
+        e2: Entry,
+        data: usize,
+    }
+
+    impl DE {
+        fn new(data: usize) -> Self {
+            Self {
+                e1: Entry::default(),
+                e2: Entry::default(),
+                data,
+            }
+        }
+    }
+
+    impl Drop for DE {
+        fn drop(&mut self) {
+            self.data = 0
+        }
+    }
+
+    intrusive_dlist! {SE, e, SEA}
+    intrusive_dlist! {DE, e1, DEA1}
+    intrusive_dlist! {DE, e2, DEA2}
+
+    #[test]
+    fn test_se_simple() {
+        unsafe {
+            let mut l: DList<SE, SEA> = DList::new();
+
+            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+
+            es.iter_mut()
+                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
+
+            assert_eq!(l.len(), 3);
+            assert_eq!(
+                vec![1, 2, 3],
+                l.iter().map(|e| e.as_ref().data).collect_vec()
+            );
+
+            drop(es);
+        }
+    }
+
+    #[test]
+    fn test_de_simple() {
+        unsafe {
+            let mut l1: DList<DE, DEA1> = DList::new();
+            let mut l2: DList<DE, DEA2> = DList::new();
+
+            let mut es = vec![DE::new(1), DE::new(2), DE::new(3)];
+            es.iter_mut()
+                .for_each(|e| l1.link_at_tail(NonNull::new_unchecked(e as *mut _)));
+            es.iter_mut()
+                .for_each(|e| l2.link_at_head(NonNull::new_unchecked(e as *mut _)));
+
+            assert_eq!(l1.len(), 3);
+            assert_eq!(l2.len(), 3);
+
+            assert_eq!(
+                vec![1, 2, 3],
+                l1.iter().map(|e| e.as_ref().data).collect_vec()
+            );
+            assert_eq!(
+                vec![3, 2, 1],
+                l2.iter().map(|e| e.as_ref().data).collect_vec()
+            );
+
+            drop(es);
+        }
+    }
+
+    #[test]
+    fn test_link_before() {
+        unsafe {
+            let mut l: DList<SE, SEA> = DList::new();
+
+            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+
+            es.iter_mut()
+                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
+
+            let mut e = SE::new(4);
+            l.link_before(
+                NonNull::new_unchecked(&mut es[2] as *mut _),
+                NonNull::new_unchecked(&mut e as *mut _),
+            );
+
+            assert_eq!(l.len(), 4);
+            assert_eq!(
+                vec![1, 2, 4, 3],
+                l.iter().map(|e| e.as_ref().data).collect_vec()
+            );
+
+            drop(e);
+            drop(es);
+        }
+    }
+
+    #[test]
+    fn test_remove() {
+        unsafe {
+            let mut l: DList<SE, SEA> = DList::new();
+
+            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+
+            es.iter_mut()
+                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
+
+            l.remove(NonNull::new_unchecked(&mut es[1] as *mut _));
+
+            assert_eq!(l.len(), 2);
+            assert_eq!(vec![1, 3], l.iter().map(|e| e.as_ref().data).collect_vec());
+
+            drop(es);
+        }
+    }
+
+    #[test]
+    fn test_link_replace() {
+        unsafe {
+            let mut l: DList<SE, SEA> = DList::new();
+
+            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+
+            es.iter_mut()
+                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
+
+            let mut e = SE::new(4);
+            l.replace(
+                NonNull::new_unchecked(&mut es[1] as *mut _),
+                NonNull::new_unchecked(&mut e as *mut _),
+            );
+
+            assert_eq!(l.len(), 3);
+            assert_eq!(
+                vec![1, 4, 3],
+                l.iter().map(|e| e.as_ref().data).collect_vec()
+            );
+
+            drop(e);
+            drop(es);
+        }
+    }
+
+    #[test]
+    fn test_move_to_head() {
+        unsafe {
+            let mut l: DList<SE, SEA> = DList::new();
+
+            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+
+            es.iter_mut()
+                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
+
+            l.move_to_head(NonNull::new_unchecked(&mut es[1] as *mut _));
+
+            assert_eq!(l.len(), 3);
+            assert_eq!(
+                vec![2, 1, 3],
+                l.iter().map(|e| e.as_ref().data).collect_vec()
+            );
+
+            drop(es);
+        }
+    }
+}
diff --git a/src/utils/mod.rs b/src/collections/mod.rs
similarity index 93%
rename from src/utils/mod.rs
rename to src/collections/mod.rs
index 269a9d39..4fe9e427 100644
--- a/src/utils/mod.rs
+++ b/src/collections/mod.rs
@@ -12,5 +12,4 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-pub mod count_min_sketch;
-pub mod hash;
+pub mod dlist;
diff --git a/src/lib.rs b/src/lib.rs
index e27f3cba..e9f6d07c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,4 +12,4 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-pub mod utils;
+pub mod collections;
diff --git a/src/utils/count_min_sketch.rs b/src/utils/count_min_sketch.rs
deleted file mode 100644
index 1bda3d03..00000000
--- a/src/utils/count_min_sketch.rs
+++ /dev/null
@@ -1,357 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//! A probabilistic counting data structure that never undercounts items before
-//! it hits counter's capacity. It is a table structure with the depth being the
-//! number of hashes and the width being the number of unique items. When a key
-//! is inserted, each row's hash function is used to generate the index for that
-//! row. Then the element's count at that index is incremented. As a result, one
-//! key being inserted will increment different indicies in each row. Querying the
-//! count returns the minimum values of these elements since some hashes might collide.
-//!
-//!
-//! Users are supposed to synchronize concurrent accesses to the data structure.
-//!
-//! E.g. insert(1)
-//! hash1(1) = 2 -> increment row 1, index 2
-//! hash2(1) = 5 -> increment row 2, index 5
-//! hash3(1) = 3 -> increment row 3, index 3
-//! etc.
-
-use crate::utils::hash::{combine_hashes, twang_mix64};
-use paste::paste;
-
-macro_rules! for_all_uint_types {
-    ($macro:ident) => {
-        $macro! {
-            {u8, U8},
-            {u16, U16},
-            {u32, U32},
-            {u64, U64},
-            {usize, Usize},
-        }
-    };
-}
-
-macro_rules! cm_sketch {
-    ($( {$type:ty, $suffix:ident}, )*) => {
-        paste! {
-            $(
-                pub struct [<CMSketch $suffix>] {
-                    width: usize,
-                    depth: usize,
-                    saturated: usize,
-
-                    /// size = width * depth
-                    table: Vec<$type>
-                }
-
-                impl [<CMSketch $suffix>] {
-
-                    /// `error`: Tolerable error in count given as a fraction of the total number of inserts. Must be between 0 and 1.
-                    /// `probability`: The certainty that the count is with in the error threshold.
-                    /// `max_width`: Maximum number of the elements per row in the table. 0 represents there is no limitations.
-                    /// `max_depth`: Maximum num of rows. 0 represents there is no limitations.
-                    pub fn new(error: f64, probability: f64, max_width: usize, max_depth: usize) -> Self {
-                        let width = Self::calculate_width(error, max_width);
-                        let depth = Self::calculate_depth(probability, max_depth);
-
-                        Self::new_with_size(width, depth)
-                    }
-
-                    pub fn new_with_size(width: usize, depth: usize) -> Self {
-                        assert!(width > 0, "Width must be greater than 0, width: {}.", width);
-                        assert!(depth > 0, "Depth must be greater than 0, depth: {}.", depth);
-
-                        Self {
-                            width,
-                            depth,
-                            saturated: 0,
-
-                            table: vec![0; width * depth],
-                        }
-                    }
-
-                    pub fn add(&mut self, key: u64) {
-                        for i in 0..self.depth {
-                            let index = self.index(i, key);
-                            if self.table[index] < self.max_count() {
-                                self.table[index] += 1;
-                                if self.table[index] == self.max_count() {
-                                    self.saturated += 1;
-                                }
-                            }
-                            // println!("(ins) table[{}, {}, {}] = {}", i, index, key, self.table[index]);
-                        }
-                    }
-
-                    pub fn count(&self, key: u64) -> $type {
-                        (0..self.depth)
-                            .into_iter()
-                            .map(|i| self.table[self.index(i, key)])
-                            // .map(|i| {
-                            //     let index = self.index(i, key);
-                            //     let res = self.table[index];
-                            //     println!("(cnt) table[{}, {}, {}] = {}", i, index, key, res);
-                            //     res
-                            // })
-                            .min()
-                            .unwrap()
-                    }
-
-                    pub fn remove(&mut self, key: u64) {
-                        let count = self.count(key);
-                        for i in 0..self.depth {
-                            let index = self.index(i, key);
-                            self.table[index] -= count;
-                        }
-                    }
-
-                    pub fn clear(&mut self) {
-                        self.saturated = 0;
-                        self.table.iter_mut().for_each(|c| *c = 0);
-                    }
-
-                    pub fn decay(&mut self, decay: f64) {
-                        self.table.iter_mut().for_each(|c| *c = (*c as f64 * decay) as $type);
-                    }
-
-                    pub fn width(&self) -> usize {
-                        self.width
-                    }
-
-                    pub fn depth(&self) -> usize {
-                        self.depth
-                    }
-
-                    pub fn max_count(&self) -> $type {
-                        $type::MAX
-                    }
-
-                    pub fn saturated(&self) -> usize {
-                        self.saturated
-                    }
-
-                    /// `max_width == 0` represents there is no limitations.
-                    fn calculate_width(error: f64, max_width: usize) -> usize {
-                        assert!(error > 0.0 && error < 1.0, "Error should be greater than 0 and less than 1, error: {}.", error);
-
-                        // From "Approximating Data with the Count-Min Data Structure" (Cormode & Muthukrishnan)
-                        let width = (2.0 / error).ceil() as usize;
-                        if max_width > 0 {
-                            std::cmp::min(width, max_width)
-                        }else {
-                            width
-                        }
-                    }
-
-                    /// `max_depth == 0` represents there is no limitations.
-                    fn calculate_depth(probability: f64, max_depth: usize) -> usize {
-                        assert!(probability > 0.0 && probability < 1.0, "Probability should be greater than 0 and less than 1, probability: {}.", probability);
-
-                        // From "Approximating Data with the Count-Min Data Structure" (Cormode & Muthukrishnan)
-                        let depth  = (1.0 - probability).log2().abs().ceil() as usize;
-                        let depth = std::cmp::max(1, depth);
-                        if max_depth > 0 {
-                            std::cmp::min(depth, max_depth)
-                        } else {
-                            depth
-                        }
-                    }
-
-                    fn index(&self, hash_index: usize, key: u64) -> usize {
-                        hash_index as usize * self.width
-                            + (combine_hashes(twang_mix64(hash_index as u64), key) as usize % self.width)
-                    }
-                }
-            )*
-        }
-    };
-}
-
-for_all_uint_types! {cm_sketch}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use rand_mt::Mt64;
-
-    macro_rules! test_cm_sketch {
-        ($( {$type:ty, $suffix:ident}, )*) => {
-            paste! {
-                $(
-                    #[test]
-                    fn [<test_add_ $type>]() {
-                        let mut cms = [<CMSketch $suffix>]::new_with_size(100, 3);
-                        let mut rng = Mt64::new_unseeded();
-                        let mut keys = vec![];
-
-                        for i in 0..10 {
-                            keys.push(rng.next_u64());
-                            for _ in 0..i {
-                                cms.add(keys[i]);
-                            }
-                        }
-
-                        for i in 0..10 {
-                            assert!(
-                                cms.count(keys[i]) >= std::cmp::min(i as $type, cms.max_count()),
-                                "assert {} >= {} failed",
-                                cms.count(keys[i]), std::cmp::min(i as $type, cms.max_count())
-                            );
-                        }
-                    }
-
-                    #[test]
-                    fn [<test_remove_ $type>]() {
-                        let mut cms = [<CMSketch $suffix>]::new_with_size(100, 3);
-                        let mut rng = Mt64::new_unseeded();
-                        let mut keys = vec![];
-
-                        for i in 0..10 {
-                            keys.push(rng.next_u64());
-                            for _ in 0..i {
-                                cms.add(keys[i]);
-                            }
-                        }
-
-                        for i in 0..10 {
-                            cms.remove(keys[i]);
-                            assert_eq!(cms.count(keys[i]), 0);
-                        }
-                    }
-
-                    #[test]
-                    fn [<test_clear_ $type>]() {
-                        let mut cms = [<CMSketch $suffix>]::new_with_size(100, 3);
-                        let mut rng = Mt64::new_unseeded();
-                        let mut keys = vec![];
-
-                        for i in 0..10 {
-                            keys.push(rng.next_u64());
-                            for _ in 0..i {
-                                cms.add(keys[i]);
-                            }
-                        }
-
-                        cms.clear();
-                        for i in 0..10 {
-                            assert_eq!(cms.count(keys[i]), 0);
-                        }
-                    }
-
-                    #[test]
-                    fn [<test_collisions_ $type>]() {
-                        let mut cms = [<CMSketch $suffix>]::new_with_size(40, 5);
-                        let mut rng = Mt64::new_unseeded();
-                        let mut keys = vec![];
-                        let mut sum = 0;
-
-                        // Try inserting more keys than cms table width
-                        for i in 0..55 {
-                            keys.push(rng.next_u64());
-                            for _ in 0..i {
-                                cms.add(keys[i]);
-                            }
-                            sum += i;
-                        }
-
-                        let error = sum as f64 * 0.05;
-                        for i in 0..10 {
-                            assert!(cms.count(keys[i]) >= i as $type);
-                            assert!(i as f64 + error >= cms.count(keys[i]) as f64);
-                        }
-                    }
-
-                    #[test]
-                    fn [<test_probability_constructor_ $type>]() {
-                        let cms = [<CMSketch $suffix>]::new(0.01, 0.95, 0, 0);
-                        assert_eq!(cms.width(), 200);
-                        assert_eq!(cms.depth(), 5);
-                    }
-
-
-                    #[test]
-                    #[should_panic]
-                    fn [<test_invalid_args_ $type>]() {
-                        [<CMSketch $suffix>]::new(0f64, 0f64, 0, 0);
-                    }
-
-                    #[test]
-                    fn [<test_decay_ $type>]() {
-                        let mut cms = [<CMSketch $suffix>]::new_with_size(100, 3);
-                        let mut rng = Mt64::new_unseeded();
-                        let mut keys = vec![];
-
-                        for i in 0..1000 {
-                            keys.push(rng.next_u64());
-                            for _ in 0..i {
-                                cms.add(keys[i]);
-                            }
-                        }
-
-                        for i in 0..1000 {
-                            assert!(
-                                cms.count(keys[i]) >= std::cmp::min(i as $type, cms.max_count()),
-                                "assert {} >= {} failed",
-                                cms.count(keys[i]), std::cmp::min(i as $type, cms.max_count())
-                            );
-                        }
-
-                        const FACTOR: f64 = 0.5;
-                        cms.decay(FACTOR);
-
-                        for i in 0..1000 {
-                            assert!(cms.count(keys[i]) >= (std::cmp::min(i as $type, cms.max_count()) as f64 * FACTOR).floor() as $type);
-                        }
-                    }
-
-                    #[test]
-                    fn [<test_overflow_ $type>]() {
-                        let mut cms = [<CMSketch $suffix>]::new_with_size(10, 3);
-                        let mut rng = Mt64::new_unseeded();
-                        let key = rng.next_u64();
-                        let max = cms.max_count();
-
-                        // Skip test if max is too large.
-                        if (max as usize) < (u32::MAX as usize) {
-                            for _ in 0..max {
-                                cms.add(key);
-                            }
-
-                            assert_eq!(cms.count(key), max);
-                            assert_eq!(cms.saturated(), 3);
-                        }
-                    }
-                )*
-            }
-        };
-    }
-
-    for_all_uint_types! {test_cm_sketch}
-}
diff --git a/src/utils/hash.rs b/src/utils/hash.rs
deleted file mode 100644
index a68a0f8d..00000000
--- a/src/utils/hash.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-/// Reduce two 64-bit hashes into one.
-///
-/// Ported from CacheLib, which uses the `Hash128to64` function from Google's city hash.
-pub fn combine_hashes(upper: u64, lower: u64) -> u64 {
-    const MUL: u64 = 0x9ddfea08eb382d69;
-
-    let mut a = (lower ^ upper).wrapping_mul(MUL);
-    a ^= a >> 47;
-    let mut b = (upper ^ a).wrapping_mul(MUL);
-    b ^= b >> 47;
-    b = b.wrapping_mul(MUL);
-    b
-}
-
-pub fn twang_mix64(val: u64) -> u64 {
-    let mut val = (!val).wrapping_add(val << 21); // val *= (1 << 21); val -= 1
-    val = val ^ (val >> 24);
-    val = val.wrapping_add(val << 3).wrapping_add(val << 8); // val *= 1 + (1 << 3) + (1 << 8)
-    val = val ^ (val >> 14);
-    val = val.wrapping_add(val << 2).wrapping_add(val << 4); // va; *= 1 + (1 << 2) + (1 << 4)
-    val = val ^ (val >> 28);
-    val = val.wrapping_add(val << 31); // val *= 1 + (1 << 31)
-    val
-}

From c74657e2a406d736fdae819f71846544c817d368 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 18 May 2023 14:53:17 +0800
Subject: [PATCH 002/261] chore: set up ci (#3)

* chore: set up ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* make clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/semantic.yml               | 16 ++++++
 .github/template/generate.sh       | 36 ++++++++++++
 .github/template/main-override.yml |  6 ++
 .github/template/pr-override.yml   |  9 +++
 .github/template/template.yml      | 72 ++++++++++++++++++++++++
 .github/workflows/main.yml         | 86 +++++++++++++++++++++++++++++
 .github/workflows/pull-request.yml | 88 ++++++++++++++++++++++++++++++
 Cargo.toml                         |  1 -
 src/collections/dlist.rs           | 74 ++++++++++++++++---------
 9 files changed, 362 insertions(+), 26 deletions(-)
 create mode 100644 .github/semantic.yml
 create mode 100755 .github/template/generate.sh
 create mode 100644 .github/template/main-override.yml
 create mode 100644 .github/template/pr-override.yml
 create mode 100644 .github/template/template.yml
 create mode 100644 .github/workflows/main.yml
 create mode 100644 .github/workflows/pull-request.yml

diff --git a/.github/semantic.yml b/.github/semantic.yml
new file mode 100644
index 00000000..f34b2e36
--- /dev/null
+++ b/.github/semantic.yml
@@ -0,0 +1,16 @@
+# Ref: https://github.com/zeke/semantic-pull-requests#configuration .
+titleAndCommits: true
+anyCommit: true
+types:
+  - feat
+  - fix
+  - docs
+  - style
+  - refactor
+  - perf
+  - test
+  - build
+  - ci
+  - chore
+  - revert
+allowMergeCommits: true
\ No newline at end of file
diff --git a/.github/template/generate.sh b/.github/template/generate.sh
new file mode 100755
index 00000000..57a5b23e
--- /dev/null
+++ b/.github/template/generate.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR"
+
+# You will need to install yq >= 4.16 to use this tool.
+# brew install yq
+
+HEADER="""
+# ================= THIS FILE IS AUTOMATICALLY GENERATED =================
+#
+# Please run generate.sh and commit after editing the workflow templates.
+#
+# ========================================================================
+"""
+
+# Generate workflow for main branch
+echo "$HEADER" > ../workflows/main.yml
+# shellcheck disable=SC2016
+yq ea '. as $item ireduce ({}; . * $item )' template.yml main-override.yml | yq eval '... comments=""' - >> ../workflows/main.yml
+echo "$HEADER" >> ../workflows/main.yml
+
+# Generate workflow for pull requests
+echo "$HEADER" > ../workflows/pull-request.yml
+# shellcheck disable=SC2016
+yq ea '. as $item ireduce ({}; . * $item )' template.yml pr-override.yml | yq eval '... comments=""' - >> ../workflows/pull-request.yml
+echo "$HEADER" >> ../workflows/pull-request.yml
+
+if [ "$1" == "--check" ] ; then
+ if ! git diff --exit-code; then
+    echo "Please run generate.sh and commit after editing the workflow templates."
+    exit 1
+ fi
+fi
diff --git a/.github/template/main-override.yml b/.github/template/main-override.yml
new file mode 100644
index 00000000..38fd29e7
--- /dev/null
+++ b/.github/template/main-override.yml
@@ -0,0 +1,6 @@
+name: CI (main)
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
diff --git a/.github/template/pr-override.yml b/.github/template/pr-override.yml
new file mode 100644
index 00000000..f8c84881
--- /dev/null
+++ b/.github/template/pr-override.yml
@@ -0,0 +1,9 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: environment-${{ github.ref }}
+  cancel-in-progress: true
diff --git a/.github/template/template.yml b/.github/template/template.yml
new file mode 100644
index 00000000..eba6475d
--- /dev/null
+++ b/.github/template/template.yml
@@ -0,0 +1,72 @@
+name:
+
+on:
+
+env:
+  RUST_TOOLCHAIN: nightly-2023-04-07
+  CARGO_TERM_COLOR: always
+  CACHE_KEY_SUFFIX: 20230518
+  RUNKV_CI: true
+
+jobs:
+  misc-check:
+    name: misc check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install tools
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
+        env:
+          YQ_VERSION: v4.16.1
+          BINARY: yq_linux_amd64
+          BUF_VERSION: 1.0.0-rc6
+      - name: Check if CI workflows are up-to-date
+        run: |
+          ./.github/template/generate.sh --check
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+  rust-test:
+    name: rust test with codecov
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy, llvm-tools-preview
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Install cargo-sort
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          cargo install cargo-sort
+      - name: Run rust cargo-sort check
+        run: |
+          cargo sort -w -c
+      - name: Run rust format check
+        run: |
+          cargo fmt --all -- --check
+      - name: Run rust clippy check
+        run: |
+          # If new CI checks are added, the one with `--locked` must be run first.
+          cargo clippy --all-targets --locked -- -D warnings
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust test with coverage
+        run: |
+          cargo llvm-cov nextest --lcov --output-path lcov.info
+      - uses: codecov/codecov-action@v2
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 00000000..cf0e635f
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,86 @@
+
+# ================= THIS FILE IS AUTOMATICALLY GENERATED =================
+#
+# Please run generate.sh and commit after editing the workflow templates.
+#
+# ========================================================================
+
+name: CI (main)
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+env:
+  RUST_TOOLCHAIN: nightly-2023-04-07
+  CARGO_TERM_COLOR: always
+  CACHE_KEY_SUFFIX: 20230518
+  RUNKV_CI: true
+jobs:
+  misc-check:
+    name: misc check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install tools
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
+        env:
+          YQ_VERSION: v4.16.1
+          BINARY: yq_linux_amd64
+          BUF_VERSION: 1.0.0-rc6
+      - name: Check if CI workflows are up-to-date
+        run: |
+          ./.github/template/generate.sh --check
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+  rust-test:
+    name: rust test with codecov
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy, llvm-tools-preview
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Install cargo-sort
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          cargo install cargo-sort
+      - name: Run rust cargo-sort check
+        run: |
+          cargo sort -w -c
+      - name: Run rust format check
+        run: |
+          cargo fmt --all -- --check
+      - name: Run rust clippy check
+        run: |
+          # If new CI checks are added, the one with `--locked` must be run first.
+          cargo clippy --all-targets --locked -- -D warnings
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust test with coverage
+        run: |
+          cargo llvm-cov nextest --lcov --output-path lcov.info
+      - uses: codecov/codecov-action@v2
+
+# ================= THIS FILE IS AUTOMATICALLY GENERATED =================
+#
+# Please run generate.sh and commit after editing the workflow templates.
+#
+# ========================================================================
+
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
new file mode 100644
index 00000000..2463a7ef
--- /dev/null
+++ b/.github/workflows/pull-request.yml
@@ -0,0 +1,88 @@
+
+# ================= THIS FILE IS AUTOMATICALLY GENERATED =================
+#
+# Please run generate.sh and commit after editing the workflow templates.
+#
+# ========================================================================
+
+name: CI
+on:
+  pull_request:
+    branches: [main]
+env:
+  RUST_TOOLCHAIN: nightly-2023-04-07
+  CARGO_TERM_COLOR: always
+  CACHE_KEY_SUFFIX: 20230518
+  RUNKV_CI: true
+jobs:
+  misc-check:
+    name: misc check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install tools
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
+        env:
+          YQ_VERSION: v4.16.1
+          BINARY: yq_linux_amd64
+          BUF_VERSION: 1.0.0-rc6
+      - name: Check if CI workflows are up-to-date
+        run: |
+          ./.github/template/generate.sh --check
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+  rust-test:
+    name: rust test with codecov
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy, llvm-tools-preview
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Install cargo-sort
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          cargo install cargo-sort
+      - name: Run rust cargo-sort check
+        run: |
+          cargo sort -w -c
+      - name: Run rust format check
+        run: |
+          cargo fmt --all -- --check
+      - name: Run rust clippy check
+        run: |
+          # If new CI checks are added, the one with `--locked` must be run first.
+          cargo clippy --all-targets --locked -- -D warnings
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust test with coverage
+        run: |
+          cargo llvm-cov nextest --lcov --output-path lcov.info
+      - uses: codecov/codecov-action@v2
+concurrency:
+  group: environment-${{ github.ref }}
+  cancel-in-progress: true
+
+# ================= THIS FILE IS AUTOMATICALLY GENERATED =================
+#
+# Please run generate.sh and commit after editing the workflow templates.
+#
+# ========================================================================
+
diff --git a/Cargo.toml b/Cargo.toml
index 24f6fcff..83edab7f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,7 +5,6 @@ edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
 license = "Apache-2.0"
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
diff --git a/src/collections/dlist.rs b/src/collections/dlist.rs
index 30d57d03..fe25924c 100644
--- a/src/collections/dlist.rs
+++ b/src/collections/dlist.rs
@@ -359,12 +359,12 @@ mod tests {
     use itertools::Itertools;
 
     #[derive(Debug)]
-    struct SE {
+    struct SingleElement {
         e: Entry,
         data: usize,
     }
 
-    impl SE {
+    impl SingleElement {
         fn new(data: usize) -> Self {
             Self {
                 e: Entry::default(),
@@ -373,20 +373,20 @@ mod tests {
         }
     }
 
-    impl Drop for SE {
+    impl Drop for SingleElement {
         fn drop(&mut self) {
             self.data = 0
         }
     }
 
     #[derive(Debug)]
-    struct DE {
+    struct DoubleElement {
         e1: Entry,
         e2: Entry,
         data: usize,
     }
 
-    impl DE {
+    impl DoubleElement {
         fn new(data: usize) -> Self {
             Self {
                 e1: Entry::default(),
@@ -396,22 +396,26 @@ mod tests {
         }
     }
 
-    impl Drop for DE {
+    impl Drop for DoubleElement {
         fn drop(&mut self) {
             self.data = 0
         }
     }
 
-    intrusive_dlist! {SE, e, SEA}
-    intrusive_dlist! {DE, e1, DEA1}
-    intrusive_dlist! {DE, e2, DEA2}
+    intrusive_dlist! {SingleElement, e, SingleElementAdapter}
+    intrusive_dlist! {DoubleElement, e1, DoubleElementAdapter1}
+    intrusive_dlist! {DoubleElement, e2, DEA2DoubleElementAdapter2}
 
     #[test]
     fn test_se_simple() {
         unsafe {
-            let mut l: DList<SE, SEA> = DList::new();
+            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
 
-            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+            let mut es = vec![
+                SingleElement::new(1),
+                SingleElement::new(2),
+                SingleElement::new(3),
+            ];
 
             es.iter_mut()
                 .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
@@ -429,10 +433,14 @@ mod tests {
     #[test]
     fn test_de_simple() {
         unsafe {
-            let mut l1: DList<DE, DEA1> = DList::new();
-            let mut l2: DList<DE, DEA2> = DList::new();
-
-            let mut es = vec![DE::new(1), DE::new(2), DE::new(3)];
+            let mut l1: DList<DoubleElement, DoubleElementAdapter1> = DList::new();
+            let mut l2: DList<DoubleElement, DEA2DoubleElementAdapter2> = DList::new();
+
+            let mut es = vec![
+                DoubleElement::new(1),
+                DoubleElement::new(2),
+                DoubleElement::new(3),
+            ];
             es.iter_mut()
                 .for_each(|e| l1.link_at_tail(NonNull::new_unchecked(e as *mut _)));
             es.iter_mut()
@@ -457,14 +465,18 @@ mod tests {
     #[test]
     fn test_link_before() {
         unsafe {
-            let mut l: DList<SE, SEA> = DList::new();
+            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
 
-            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+            let mut es = vec![
+                SingleElement::new(1),
+                SingleElement::new(2),
+                SingleElement::new(3),
+            ];
 
             es.iter_mut()
                 .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
 
-            let mut e = SE::new(4);
+            let mut e = SingleElement::new(4);
             l.link_before(
                 NonNull::new_unchecked(&mut es[2] as *mut _),
                 NonNull::new_unchecked(&mut e as *mut _),
@@ -484,9 +496,13 @@ mod tests {
     #[test]
     fn test_remove() {
         unsafe {
-            let mut l: DList<SE, SEA> = DList::new();
+            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
 
-            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+            let mut es = vec![
+                SingleElement::new(1),
+                SingleElement::new(2),
+                SingleElement::new(3),
+            ];
 
             es.iter_mut()
                 .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
@@ -503,14 +519,18 @@ mod tests {
     #[test]
     fn test_link_replace() {
         unsafe {
-            let mut l: DList<SE, SEA> = DList::new();
+            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
 
-            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+            let mut es = vec![
+                SingleElement::new(1),
+                SingleElement::new(2),
+                SingleElement::new(3),
+            ];
 
             es.iter_mut()
                 .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
 
-            let mut e = SE::new(4);
+            let mut e = SingleElement::new(4);
             l.replace(
                 NonNull::new_unchecked(&mut es[1] as *mut _),
                 NonNull::new_unchecked(&mut e as *mut _),
@@ -530,9 +550,13 @@ mod tests {
     #[test]
     fn test_move_to_head() {
         unsafe {
-            let mut l: DList<SE, SEA> = DList::new();
+            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
 
-            let mut es = vec![SE::new(1), SE::new(2), SE::new(3)];
+            let mut es = vec![
+                SingleElement::new(1),
+                SingleElement::new(2),
+                SingleElement::new(3),
+            ];
 
             es.iter_mut()
                 .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));

From 86513b0299dfaf07fe3670873ec233bd63bfed72 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 18 May 2023 18:40:48 +0800
Subject: [PATCH 003/261] feat: inroduce lru policy (#4)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .gitignore               |   2 +
 src/collections/dlist.rs |  26 ++++
 src/lib.rs               |   1 +
 src/policies/lru.rs      | 301 +++++++++++++++++++++++++++++++++++++++
 src/policies/mod.rs      |  41 ++++++
 5 files changed, 371 insertions(+)
 create mode 100644 src/policies/lru.rs
 create mode 100644 src/policies/mod.rs

diff --git a/.gitignore b/.gitignore
index ea8c4bf7..e040625b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
+/.vscode
+
 /target
diff --git a/src/collections/dlist.rs b/src/collections/dlist.rs
index fe25924c..b6470b69 100644
--- a/src/collections/dlist.rs
+++ b/src/collections/dlist.rs
@@ -279,6 +279,30 @@ impl<E, A: Adapter<E>> DList<E, A> {
         self.link_at_tail(element);
     }
 
+    pub fn head(&self) -> Option<NonNull<E>> {
+        self.head.map(|entry| A::en2el(entry))
+    }
+
+    pub fn tail(&self) -> Option<NonNull<E>> {
+        self.tail.map(|entry| A::en2el(entry))
+    }
+
+    /// # Safety
+    ///
+    /// element` must be in the list
+    pub unsafe fn prev(&self, element: NonNull<E>) -> Option<NonNull<E>> {
+        let entry = A::el2en(element);
+        entry.as_ref().prev.map(|entry| A::en2el(entry))
+    }
+
+    /// # Safety
+    ///
+    /// element` must be in the list
+    pub unsafe fn next(&self, element: NonNull<E>) -> Option<NonNull<E>> {
+        let entry = A::el2en(element);
+        entry.as_ref().next.map(|entry| A::en2el(entry))
+    }
+
     pub fn len(&self) -> usize {
         self.len
     }
@@ -298,6 +322,8 @@ impl<E, A: Adapter<E>> DList<E, A> {
     }
 }
 
+pub trait DListExt {}
+
 pub struct Iter<'a, E, A: Adapter<E>> {
     dlist: &'a mut DList<E, A>,
     entry: Option<NonNull<Entry>>,
diff --git a/src/lib.rs b/src/lib.rs
index e9f6d07c..bf73502a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,3 +13,4 @@
 //  limitations under the License.
 
 pub mod collections;
+pub mod policies;
diff --git a/src/policies/lru.rs b/src/policies/lru.rs
new file mode 100644
index 00000000..01399bbb
--- /dev/null
+++ b/src/policies/lru.rs
@@ -0,0 +1,301 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::ptr::NonNull;
+use std::time::SystemTime;
+
+use crate::collections::dlist::{DList, Entry, Iter};
+use crate::intrusive_dlist;
+
+use super::{AccessMode, Index};
+
+pub struct Config {
+    update_on_write: bool,
+
+    update_on_read: bool,
+
+    /// Insertion point of the new entry, between 0 and 1.
+    lru_insertion_point_fraction: f64,
+}
+
+pub struct Handle<I: Index> {
+    entry: Entry,
+
+    is_in_cache: bool,
+    is_accessed: bool,
+    update_time: SystemTime,
+
+    is_in_tail: bool,
+
+    index: I,
+}
+
+impl<I: Index> Handle<I> {
+    pub fn new(index: I) -> Self {
+        Self {
+            entry: Entry::default(),
+
+            is_in_cache: false,
+            is_accessed: false,
+            update_time: SystemTime::now(),
+
+            is_in_tail: false,
+
+            index,
+        }
+    }
+}
+
+intrusive_dlist! { Handle<I: Index>, entry, HandleDListAdapter}
+
+pub struct Lru<I: Index> {
+    /// lru list
+    lru: DList<Handle<I>, HandleDListAdapter>,
+
+    /// insertion point
+    insertion_point: Option<NonNull<Handle<I>>>,
+
+    /// length of tail after insertion point
+    tail_len: usize,
+
+    config: Config,
+}
+
+impl<I: Index> Lru<I> {
+    pub fn new(config: Config) -> Self {
+        Self {
+            lru: DList::new(),
+
+            insertion_point: None,
+
+            tail_len: 0,
+
+            config,
+        }
+    }
+
+    /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
+    /// returns `false` otherwise.
+    pub fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
+        unsafe {
+            if (mode == AccessMode::Read && !self.config.update_on_read)
+                || (mode == AccessMode::Write && !self.config.update_on_write)
+            {
+                return false;
+            }
+            handle.as_mut().is_accessed = true;
+
+            // TODO(MrCroxx): try trigger reconfigure
+
+            self.ensuer_not_insertion_point(handle);
+
+            if handle.as_ref().is_in_cache {
+                self.lru.move_to_head(handle);
+                handle.as_mut().update_time = SystemTime::now();
+            }
+
+            if handle.as_ref().is_in_tail {
+                handle.as_mut().is_in_tail = false;
+                self.tail_len -= 1;
+                self.update_lru_insertion_point();
+            }
+
+            true
+        }
+    }
+
+    /// Returns `true` if handle is successfully added into the lru,
+    /// returns `false` if the handle is already in the lru.
+    pub fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+        unsafe {
+            if handle.as_ref().is_in_cache {
+                return false;
+            }
+
+            match self.insertion_point {
+                Some(insertion_point) => self.lru.link_before(insertion_point, handle),
+                None => self.lru.link_at_head(handle),
+            }
+            handle.as_mut().is_in_cache = true;
+            handle.as_mut().update_time = SystemTime::now();
+            handle.as_mut().is_accessed = false;
+            self.update_lru_insertion_point();
+
+            true
+        }
+    }
+
+    /// Returns `true` if handle is successfully removed from the lru,
+    /// returns `false` if the handle is unchanged.
+    pub fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+        unsafe {
+            if !handle.as_ref().is_in_cache {
+                return false;
+            }
+
+            self.ensuer_not_insertion_point(handle);
+            self.lru.remove(handle);
+            handle.as_mut().is_accessed = false;
+            if handle.as_ref().is_in_tail {
+                handle.as_mut().is_in_tail = false;
+                self.tail_len -= 1;
+            }
+
+            true
+        }
+    }
+
+    pub fn eviction_iter(&mut self) -> EvictionIter<'_, I> {
+        unsafe {
+            let mut iter = self.lru.iter();
+            iter.tail();
+            EvictionIter { iter }
+        }
+    }
+
+    fn update_lru_insertion_point(&mut self) {
+        unsafe {
+            if self.config.lru_insertion_point_fraction == 0.0 {
+                return;
+            }
+
+            if self.insertion_point.is_none() {
+                self.insertion_point = self.lru.tail();
+                self.tail_len = 0;
+                if let Some(insertion_point) = &mut self.insertion_point {
+                    insertion_point.as_mut().is_in_tail = true;
+                    self.tail_len += 1;
+                }
+            }
+
+            if self.lru.len() <= 1 {
+                return;
+            }
+
+            assert!(self.insertion_point.is_some());
+
+            let expected_tail_len =
+                (self.lru.len() as f64 * (1.0 - self.config.lru_insertion_point_fraction)) as usize;
+
+            let mut curr = self.insertion_point.unwrap();
+            while self.tail_len < expected_tail_len && Some(curr) != self.lru.head() {
+                curr = self.lru.prev(curr).unwrap();
+                curr.as_mut().is_in_tail = true;
+                self.tail_len += 1;
+            }
+            while self.tail_len > expected_tail_len && Some(curr) != self.lru.tail() {
+                curr.as_mut().is_in_tail = false;
+                self.tail_len -= 1;
+                curr = self.lru.next(curr).unwrap();
+            }
+
+            self.insertion_point = Some(curr);
+        }
+    }
+
+    fn ensuer_not_insertion_point(&mut self, handle: NonNull<Handle<I>>) {
+        unsafe {
+            if Some(handle) == self.insertion_point {
+                self.insertion_point = self.lru.prev(handle);
+                match &mut self.insertion_point {
+                    Some(insertion_point) => {
+                        self.tail_len += 1;
+                        insertion_point.as_mut().is_in_tail = true;
+                    }
+                    // TODO(MrCroxx): think ?
+                    None => assert_eq!(self.lru.len(), 1),
+                }
+            }
+        }
+    }
+}
+
+pub struct EvictionIter<'a, I: Index> {
+    iter: Iter<'a, Handle<I>, HandleDListAdapter>,
+}
+
+impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
+    type Item = &'a I;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        unsafe {
+            match self.iter.element() {
+                Some(element) => {
+                    self.iter.prev();
+                    Some(&element.as_ref().index)
+                }
+                None => None,
+            }
+        }
+    }
+}
+
+// unsafe impl `Send + Sync` for `Lru` because it uses `NonNull`
+unsafe impl<I: Index> Send for Lru<I> {}
+unsafe impl<I: Index> Sync for Lru<I> {}
+
+#[cfg(test)]
+mod tests {
+    use itertools::Itertools;
+
+    use super::*;
+
+    impl Index for u64 {}
+
+    fn ptr<I: Index>(handle: &mut Handle<I>) -> NonNull<Handle<I>> {
+        unsafe { NonNull::new_unchecked(handle as *mut _) }
+    }
+
+    #[test]
+    fn test_lru_simple() {
+        let config = Config {
+            update_on_write: true,
+            update_on_read: true,
+            lru_insertion_point_fraction: 0.0,
+        };
+        let mut lru: Lru<u64> = Lru::new(config);
+
+        let mut handles = vec![Handle::new(0), Handle::new(1), Handle::new(2)];
+
+        lru.add(ptr(&mut handles[0]));
+        lru.add(ptr(&mut handles[1]));
+        lru.add(ptr(&mut handles[2]));
+
+        assert_eq!(vec![0, 1, 2], lru.eviction_iter().copied().collect_vec());
+
+        lru.record_access(ptr(&mut handles[1]), AccessMode::Read);
+
+        assert_eq!(vec![0, 2, 1], lru.eviction_iter().copied().collect_vec());
+
+        lru.remove(ptr(&mut handles[2]));
+
+        assert_eq!(vec![0, 1], lru.eviction_iter().copied().collect_vec());
+
+        drop(handles);
+    }
+}
diff --git a/src/policies/mod.rs b/src/policies/mod.rs
new file mode 100644
index 00000000..561f7b48
--- /dev/null
+++ b/src/policies/mod.rs
@@ -0,0 +1,41 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//! A collection of cache eviction policies.
+//!
+//! Cache eviction policies only cares about the order of cached entries.
+//! They don't store the real cache entries or resource usage.
+
+pub mod lru;
+
+use std::hash::Hash;
+
+pub trait Index: Clone + Hash + Send + Sync + 'static {}
+
+pub trait Policy: Send + Sync + 'static {}
+
+#[derive(PartialEq, Eq, Debug)]
+pub enum AccessMode {
+    Read,
+    Write,
+}
+
+pub enum HandleInner<I: Index> {
+    LruHandle(lru::Handle<I>),
+}
+
+#[allow(unused)]
+pub struct Handle<I: Index> {
+    inner: HandleInner<I>,
+}

From 851b6bdfc680382c937e7ea234e0948598d3ccaf Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 19 May 2023 14:04:46 +0800
Subject: [PATCH 004/261] feat: introduce TinyLfu eviction policy (#5)

* feat: introduce TinyLfu eviction policy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* make ci happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock               |  75 +++++++
 Cargo.toml               |   2 +
 src/collections/dlist.rs |  12 +-
 src/policies/lru.rs      |   2 -
 src/policies/mod.rs      |   4 +
 src/policies/tinylfu.rs  | 455 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 542 insertions(+), 8 deletions(-)
 create mode 100644 src/policies/tinylfu.rs

diff --git a/Cargo.lock b/Cargo.lock
index 5e4c9e08..23466cba 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -20,6 +20,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cmsketch"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52c83ab4269fb46767c201bcf7957df0eb14988fbea81fef4efbc8ba8f0241ef"
+dependencies = [
+ "paste",
+]
+
 [[package]]
 name = "crossbeam-epoch"
 version = "0.9.14"
@@ -52,6 +61,7 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
 name = "foyer"
 version = "0.1.0"
 dependencies = [
+ "cmsketch",
  "crossbeam-epoch",
  "crossbeam-utils",
  "futures",
@@ -60,6 +70,7 @@ dependencies = [
  "parking_lot",
  "paste",
  "rand_mt",
+ "twox-hash",
 ]
 
 [[package]]
@@ -151,6 +162,17 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -232,6 +254,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.56"
@@ -250,11 +278,35 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom",
+]
 
 [[package]]
 name = "rand_mt"
@@ -295,6 +347,12 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "syn"
 version = "2.0.15"
@@ -306,12 +364,29 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "twox-hash"
+version = "1.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
+dependencies = [
+ "cfg-if",
+ "rand",
+ "static_assertions",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
 
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
 [[package]]
 name = "windows-sys"
 version = "0.45.0"
diff --git a/Cargo.toml b/Cargo.toml
index 83edab7f..848664f6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,12 +8,14 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+cmsketch = "0.1"
 crossbeam-epoch = "0.9"
 futures = "0.3"
 itertools = "0.10.5"
 memoffset = "0.8"
 parking_lot = "0.12"
 paste = "1.0"
+twox-hash = "1"
 
 [dev-dependencies]
 crossbeam-utils = "0.8"
diff --git a/src/collections/dlist.rs b/src/collections/dlist.rs
index b6470b69..a87b46ff 100644
--- a/src/collections/dlist.rs
+++ b/src/collections/dlist.rs
@@ -41,21 +41,21 @@ macro_rules! intrusive_dlist {
         struct $adapter;
 
         impl $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? $crate::collections::dlist::Adapter<$element $(< $( $lt ),+ >)?> for $adapter  {
-            fn en2el(entry: NonNull<Entry>) -> NonNull<$element $(< $( $lt ),+ >)?> {
+            fn en2el(entry: std::ptr::NonNull<Entry>) -> std::ptr::NonNull<$element $(< $( $lt ),+ >)?> {
                 unsafe {
                     let ptr = (entry.as_ptr() as *mut u8)
                         .sub(memoffset::offset_of!($element $(< $( $lt ),+ >)?, $entry))
                         .cast::<$element $(< $( $lt ),+ >)?>();
-                    NonNull::new_unchecked(ptr)
+                    std::ptr::NonNull::new_unchecked(ptr)
                 }
             }
 
-            fn el2en(element: NonNull<$element $(< $( $lt ),+ >)?>) -> NonNull<Entry> {
+            fn el2en(element: std::ptr::NonNull<$element $(< $( $lt ),+ >)?>) -> std::ptr::NonNull<Entry> {
                 unsafe {
                     let ptr = (element.as_ptr() as *mut u8)
                         .add(memoffset::offset_of!($element $(< $( $lt ),+ >)?, $entry))
                         .cast::<Entry>();
-                    NonNull::new_unchecked(ptr)
+                    std::ptr::NonNull::new_unchecked(ptr)
                 }
             }
         }
@@ -316,7 +316,7 @@ impl<E, A: Adapter<E>> DList<E, A> {
     /// # Safety
     ///
     /// there is no guarantee that the element cannot be modified while iterating
-    pub unsafe fn iter(&mut self) -> Iter<'_, E, A> {
+    pub unsafe fn iter(&self) -> Iter<'_, E, A> {
         let entry = self.head;
         Iter { dlist: self, entry }
     }
@@ -325,7 +325,7 @@ impl<E, A: Adapter<E>> DList<E, A> {
 pub trait DListExt {}
 
 pub struct Iter<'a, E, A: Adapter<E>> {
-    dlist: &'a mut DList<E, A>,
+    dlist: &'a DList<E, A>,
     entry: Option<NonNull<Entry>>,
 }
 
diff --git a/src/policies/lru.rs b/src/policies/lru.rs
index 01399bbb..503199ad 100644
--- a/src/policies/lru.rs
+++ b/src/policies/lru.rs
@@ -265,8 +265,6 @@ mod tests {
 
     use super::*;
 
-    impl Index for u64 {}
-
     fn ptr<I: Index>(handle: &mut Handle<I>) -> NonNull<Handle<I>> {
         unsafe { NonNull::new_unchecked(handle as *mut _) }
     }
diff --git a/src/policies/mod.rs b/src/policies/mod.rs
index 561f7b48..9bb740b3 100644
--- a/src/policies/mod.rs
+++ b/src/policies/mod.rs
@@ -18,6 +18,7 @@
 //! They don't store the real cache entries or resource usage.
 
 pub mod lru;
+pub mod tinylfu;
 
 use std::hash::Hash;
 
@@ -39,3 +40,6 @@ pub enum HandleInner<I: Index> {
 pub struct Handle<I: Index> {
     inner: HandleInner<I>,
 }
+
+#[cfg(test)]
+impl Index for u64 {}
diff --git a/src/policies/tinylfu.rs b/src/policies/tinylfu.rs
new file mode 100644
index 00000000..e0f8dc56
--- /dev/null
+++ b/src/policies/tinylfu.rs
@@ -0,0 +1,455 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::hash::Hasher;
+use std::ptr::NonNull;
+use std::time::SystemTime;
+
+use cmsketch::CMSketchUsize;
+use twox_hash::XxHash64;
+
+use crate::collections::dlist::{DList, Entry, Iter};
+use crate::intrusive_dlist;
+
+use super::{AccessMode, Index};
+
+const MIN_CAPACITY: usize = 100;
+const ERROR_THRESHOLD: f64 = 5.0;
+const HASH_COUNT: usize = 4;
+const DECAY_FACTOR: f64 = 0.5;
+
+pub struct Config {
+    update_on_write: bool,
+
+    update_on_read: bool,
+
+    /// The multiplier for window len given the cache size.
+    window_to_cache_size_ratio: usize,
+
+    /// The ratio of tiny lru capacity to overall capacity.
+    tiny_lru_capacity_ratio: f64,
+}
+
+#[derive(PartialEq, Eq, Debug)]
+enum LruType {
+    Tiny,
+    Main,
+}
+
+pub struct Handle<I: Index> {
+    entry_tiny: Entry,
+    entry_main: Entry,
+
+    is_in_cache: bool,
+    is_accessed: bool,
+    update_time: SystemTime,
+
+    lru_type: LruType,
+
+    index: I,
+}
+
+impl<I: Index> Handle<I> {
+    pub fn new(index: I) -> Self {
+        Self {
+            entry_tiny: Entry::default(),
+            entry_main: Entry::default(),
+
+            is_in_cache: false,
+            is_accessed: false,
+            update_time: SystemTime::now(),
+
+            lru_type: LruType::Tiny,
+
+            index,
+        }
+    }
+}
+
+intrusive_dlist! { Handle<I: Index>, entry_tiny, HandleDListTinyAdapter}
+intrusive_dlist! { Handle<I: Index>, entry_main, HandleDListMainAdapter}
+
+/// Implements the W-TinyLFU cache eviction policy as described in -
+///
+/// https://arxiv.org/pdf/1512.00727.pdf
+///
+/// The cache is split into 2 parts, the main cache and the tiny cache.
+/// The tiny cache is typically sized to be 1% of the total cache with
+/// the main cache being the rest 99%. Both caches are implemented using
+/// LRUs. New items land in tiny cache. During eviction, the tail item
+/// from the tiny cache is promoted to main cache if its frequency is
+/// higher than the tail item of of main cache, and the tail of main
+/// cache is evicted. This gives the frequency based admission into main
+/// cache. Hits in each cache simply move the item to the head of each
+/// LRU cache.
+/// The frequency counts are maintained in count-min-sketch approximate
+/// counters -
+///
+/// Counter Overhead:
+/// The window_to_cache_size_ratio determines the size of counters.
+/// The default value is 32 which means the counting window size is
+/// 32 times the cache size. After every 32 X cache capacity number
+/// of items, the counts are halved to weigh frequency by recency.
+/// The function counter_size() returns the size of the counters
+/// in bytes. See maybe_grow_access_counters() implementation for
+/// how the size is computed.
+///
+/// Tiny cache size:
+/// This default to 1%. There's no need to tune this parameter.
+pub struct TinyLfu<I: Index> {
+    /// tiny lru list
+    lru_tiny: DList<Handle<I>, HandleDListTinyAdapter>,
+
+    /// main lru list
+    lru_main: DList<Handle<I>, HandleDListMainAdapter>,
+
+    /// the window length counter
+    window_size: usize,
+
+    /// maxumum value of window length which when hit the counters are halved
+    max_window_size: usize,
+
+    /// the capacity for which the counters are sized
+    capacity: usize,
+
+    /// approximate streaming frequency counters
+    ///
+    /// the counts are halved every time the max_window_len is hit
+    frequencies: CMSketchUsize,
+
+    config: Config,
+}
+
+impl<I: Index> TinyLfu<I> {
+    pub fn new(config: Config) -> Self {
+        let mut res = Self {
+            lru_tiny: DList::new(),
+            lru_main: DList::new(),
+
+            window_size: 0,
+            max_window_size: 0,
+            capacity: 0,
+
+            // A dummy size, will be updated later.
+            frequencies: CMSketchUsize::new_with_size(1, 1),
+
+            config,
+        };
+        res.maybe_grow_access_counters();
+        res
+    }
+
+    /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
+    /// returns `false` otherwise.
+    pub fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
+        unsafe {
+            if (mode == AccessMode::Read && !self.config.update_on_read)
+                || (mode == AccessMode::Write && !self.config.update_on_write)
+            {
+                return false;
+            }
+            handle.as_mut().is_accessed = true;
+
+            if !handle.as_ref().is_in_cache {
+                return false;
+            }
+
+            match handle.as_ref().lru_type {
+                LruType::Tiny => self.lru_tiny.move_to_head(handle),
+                LruType::Main => self.lru_main.move_to_head(handle),
+            }
+            handle.as_mut().update_time = SystemTime::now();
+            self.update_frequencies(handle);
+
+            true
+        }
+    }
+
+    /// Returns `true` if handle is successfully added into the lru,
+    /// returns `false` if the handle is already in the lru.
+    pub fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+        unsafe {
+            if handle.as_ref().is_in_cache {
+                return false;
+            }
+
+            self.lru_tiny.link_at_head(handle);
+            handle.as_mut().lru_type = LruType::Tiny;
+
+            // Initialize the frequency count for this handle.
+            self.update_frequencies(handle);
+
+            // If tiny cache is full, unconditionally promote tail to main cache.
+            let expected_tiny_len = (self.config.tiny_lru_capacity_ratio
+                * (self.lru_tiny.len() + self.lru_main.len()) as f64)
+                as usize;
+            if self.lru_tiny.len() > expected_tiny_len {
+                let mut handle_tail = self.lru_tiny.tail().unwrap();
+                self.lru_tiny.remove(handle_tail);
+
+                self.lru_main.link_at_head(handle_tail);
+                handle_tail.as_mut().lru_type = LruType::Main;
+            } else {
+                self.maybe_promote_tail();
+            }
+
+            // If the number of counters are too small for the cache size, double them.
+            self.maybe_grow_access_counters();
+
+            handle.as_mut().is_in_cache = true;
+            handle.as_mut().is_accessed = false;
+            handle.as_mut().update_time = SystemTime::now();
+
+            true
+        }
+    }
+
+    /// Returns `true` if handle is successfully removed from the lru,
+    /// returns `false` if the handle is unchanged.
+    pub fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+        unsafe {
+            if !handle.as_ref().is_in_cache {
+                return false;
+            }
+
+            match handle.as_ref().lru_type {
+                LruType::Main => self.lru_main.remove(handle),
+                LruType::Tiny => self.lru_tiny.remove(handle),
+            }
+
+            handle.as_mut().is_accessed = false;
+            handle.as_mut().is_in_cache = false;
+
+            true
+        }
+    }
+
+    pub fn eviction_iter<'a>(&'a mut self) -> EvictionIter<'a, I> {
+        unsafe {
+            let mut iter_main: Iter<'a, _, _> = self.lru_main.iter();
+            iter_main.tail();
+            let mut iter_tiny: Iter<'a, _, _> = self.lru_tiny.iter();
+            iter_tiny.tail();
+            EvictionIter {
+                tinylfu: self,
+                iter_main,
+                iter_tiny,
+            }
+        }
+    }
+
+    fn maybe_grow_access_counters(&mut self) {
+        let capacity = self.lru_tiny.len() + self.lru_main.len();
+
+        // If the new capacity ask is more than double the current size,
+        // recreate the approximate frequency counters.
+        if 2 * self.capacity > capacity {
+            return;
+        }
+
+        self.capacity = std::cmp::max(capacity, MIN_CAPACITY);
+
+        // The window counter that's incremented on every fetch.
+        self.window_size = 0;
+
+        // The frequency counters are halved every `max_window_size` fetches to decay the frequency counts.
+        self.max_window_size = self.capacity * self.config.window_to_cache_size_ratio;
+
+        // Number of frequency counters - roughly equal to the window size divided by error tolerance.
+        let num_counters = (1f64.exp() * self.max_window_size as f64 / ERROR_THRESHOLD) as usize;
+        let num_counters = num_counters.next_power_of_two();
+
+        self.frequencies = CMSketchUsize::new_with_size(num_counters, HASH_COUNT);
+    }
+
+    fn update_frequencies(&mut self, handle: NonNull<Handle<I>>) {
+        self.frequencies.add(Self::hash_handle(handle));
+        self.window_size += 1;
+
+        // Decay counts every `max_window_size`. This avoids having items that were
+        // accessed frequently (were hot) but aren't being accessed anymore (are cold)
+        // from staying in cache forever.
+        if self.window_size == self.max_window_size {
+            self.window_size >>= 1;
+            self.frequencies.decay(DECAY_FACTOR);
+        }
+    }
+
+    fn maybe_promote_tail(&mut self) {
+        unsafe {
+            let mut handle_main = match self.lru_main.tail() {
+                Some(handle) => handle,
+                None => return,
+            };
+            let mut handle_tiny = match self.lru_tiny.tail() {
+                Some(handle) => handle,
+                None => return,
+            };
+
+            if self.admit_to_main(handle_main, handle_tiny) {
+                self.lru_tiny.remove(handle_tiny);
+                self.lru_main.link_at_head(handle_tiny);
+                handle_tiny.as_mut().lru_type = LruType::Main;
+
+                self.lru_main.remove(handle_main);
+                self.lru_tiny.link_at_tail(handle_main);
+                handle_main.as_mut().lru_type = LruType::Tiny;
+                return;
+            }
+
+            // A node with high frequency at the tail of main cache might prevent
+            // promotions from tiny cache from happening for a long time. Relocate
+            // the tail of main cache to prevent this.
+            self.lru_main.move_to_head(handle_main);
+        }
+    }
+
+    fn admit_to_main(
+        &self,
+        handle_main: NonNull<Handle<I>>,
+        handle_tiny: NonNull<Handle<I>>,
+    ) -> bool {
+        unsafe {
+            assert_eq!(handle_main.as_ref().lru_type, LruType::Main);
+            assert_eq!(handle_tiny.as_ref().lru_type, LruType::Tiny);
+
+            let frequent_main = self.frequencies.count(Self::hash_handle(handle_main));
+            let frequent_tiny = self.frequencies.count(Self::hash_handle(handle_tiny));
+
+            frequent_main <= frequent_tiny
+        }
+    }
+
+    fn hash_handle(handle: NonNull<Handle<I>>) -> u64 {
+        let mut hasher = XxHash64::default();
+        unsafe { handle.as_ref().index.hash(&mut hasher) };
+        hasher.finish()
+    }
+}
+
+pub struct EvictionIter<'a, I: Index> {
+    tinylfu: &'a TinyLfu<I>,
+    iter_main: Iter<'a, Handle<I>, HandleDListMainAdapter>,
+    iter_tiny: Iter<'a, Handle<I>, HandleDListTinyAdapter>,
+}
+
+impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
+    type Item = &'a I;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        unsafe {
+            let handle_main = self.iter_main.element();
+            let handle_tiny = self.iter_tiny.element();
+
+            match (handle_main, handle_tiny) {
+                (None, None) => None,
+                (Some(handle_main), None) => {
+                    self.iter_main.prev();
+                    Some(&handle_main.as_ref().index)
+                }
+                (None, Some(handle_tiny)) => {
+                    self.iter_tiny.prev();
+                    Some(&handle_tiny.as_ref().index)
+                }
+                (Some(handle_main), Some(handle_tiny)) => {
+                    // Eviction from tiny or main depending on whether the tiny handle woould be
+                    // admitted to main cachce. If it would be, evict from main cache, otherwise
+                    // from tiny cache.
+                    if self.tinylfu.admit_to_main(handle_main, handle_tiny) {
+                        self.iter_main.prev();
+                        Some(&handle_main.as_ref().index)
+                    } else {
+                        self.iter_tiny.prev();
+                        Some(&handle_tiny.as_ref().index)
+                    }
+                }
+            }
+        }
+    }
+}
+
+// unsafe impl `Send + Sync` for `Lru` because it uses `NonNull`
+unsafe impl<I: Index> Send for TinyLfu<I> {}
+unsafe impl<I: Index> Sync for TinyLfu<I> {}
+
+#[cfg(test)]
+mod tests {
+    use itertools::Itertools;
+
+    use super::*;
+
+    fn ptr<I: Index>(handle: &mut Handle<I>) -> NonNull<Handle<I>> {
+        unsafe { NonNull::new_unchecked(handle as *mut _) }
+    }
+
+    #[test]
+    fn test_lru_simple() {
+        let config = Config {
+            update_on_write: true,
+            update_on_read: true,
+            window_to_cache_size_ratio: 10,
+            tiny_lru_capacity_ratio: 0.01,
+        };
+        let mut lfu: TinyLfu<u64> = TinyLfu::new(config);
+
+        let mut handles = (0..101).map(Handle::new).collect_vec();
+        for handle in handles.iter_mut().take(100) {
+            lfu.add(ptr(handle));
+        }
+
+        assert_eq!(99, lfu.lru_main.len());
+        assert_eq!(1, lfu.lru_tiny.len());
+        assert_eq!(handles[0].lru_type, LruType::Tiny);
+
+        // 0 will be evicted at last because it is on tiny lru but its frequency equals to others
+        assert_eq!(
+            (1..100).chain([0].into_iter()).collect_vec(),
+            lfu.eviction_iter().copied().collect_vec()
+        );
+
+        for handle in handles.iter_mut().take(100) {
+            lfu.record_access(ptr(handle), AccessMode::Read);
+        }
+        lfu.record_access(ptr(&mut handles[0]), AccessMode::Read);
+        lfu.add(ptr(&mut handles[100]));
+
+        assert_eq!(handles[0].lru_type, LruType::Main);
+        assert_eq!(handles[100].lru_type, LruType::Tiny);
+
+        assert_eq!(
+            [100]
+                .into_iter()
+                .chain(1..100)
+                .chain([0].into_iter())
+                .collect_vec(),
+            lfu.eviction_iter().copied().collect_vec()
+        );
+
+        drop(handles);
+    }
+}

From 05346952d013ef04bc2239f9f8550ae6e0d0a921 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 19 May 2023 14:39:56 +0800
Subject: [PATCH 005/261] feat: introduce generic policy (#6)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 src/lib.rs              |  7 +++++
 src/policies/lru.rs     | 43 +++++++++++++++++++++++----
 src/policies/mod.rs     | 64 ++++++++++++++++++++++++++++++++++++-----
 src/policies/tinylfu.rs | 45 ++++++++++++++++++++++++-----
 4 files changed, 139 insertions(+), 20 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index bf73502a..e7263c75 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,5 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use policies::Policy;
+
 pub mod collections;
 pub mod policies;
+
+#[allow(unused)]
+pub struct Cache<P: Policy> {
+    policy: P,
+}
diff --git a/src/policies/lru.rs b/src/policies/lru.rs
index 503199ad..b0f96b06 100644
--- a/src/policies/lru.rs
+++ b/src/policies/lru.rs
@@ -30,9 +30,9 @@ use std::ptr::NonNull;
 use std::time::SystemTime;
 
 use crate::collections::dlist::{DList, Entry, Iter};
-use crate::intrusive_dlist;
+use crate::{extract_handle, intrusive_dlist};
 
-use super::{AccessMode, Index};
+use super::{AccessMode, Index, Policy};
 
 pub struct Config {
     update_on_write: bool,
@@ -69,6 +69,10 @@ impl<I: Index> Handle<I> {
             index,
         }
     }
+
+    pub fn index(&self) -> &I {
+        &self.index
+    }
 }
 
 intrusive_dlist! { Handle<I: Index>, entry, HandleDListAdapter}
@@ -101,7 +105,7 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
     /// returns `false` otherwise.
-    pub fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
+    fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
         unsafe {
             if (mode == AccessMode::Read && !self.config.update_on_read)
                 || (mode == AccessMode::Write && !self.config.update_on_write)
@@ -131,7 +135,7 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if handle is successfully added into the lru,
     /// returns `false` if the handle is already in the lru.
-    pub fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
             if handle.as_ref().is_in_cache {
                 return false;
@@ -152,7 +156,7 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if handle is successfully removed from the lru,
     /// returns `false` if the handle is unchanged.
-    pub fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
             if !handle.as_ref().is_in_cache {
                 return false;
@@ -170,7 +174,7 @@ impl<I: Index> Lru<I> {
         }
     }
 
-    pub fn eviction_iter(&mut self) -> EvictionIter<'_, I> {
+    fn eviction_iter(&mut self) -> EvictionIter<'_, I> {
         unsafe {
             let mut iter = self.lru.iter();
             iter.tail();
@@ -259,6 +263,33 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
 unsafe impl<I: Index> Send for Lru<I> {}
 unsafe impl<I: Index> Sync for Lru<I> {}
 
+impl<I: Index> Policy for Lru<I> {
+    type I = I;
+
+    fn add(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
+        let handle = extract_handle!(handle, Lru);
+        self.add(handle)
+    }
+
+    fn remove(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
+        let handle = extract_handle!(handle, Lru);
+        self.remove(handle)
+    }
+
+    fn record_access(
+        &mut self,
+        mut handle: NonNull<super::Handle<Self::I>>,
+        mode: AccessMode,
+    ) -> bool {
+        let handle = extract_handle!(handle, Lru);
+        self.record_access(handle, mode)
+    }
+
+    fn eviction_iter(&mut self) -> super::EvictionIter<'_, Self::I> {
+        super::EvictionIter::LruEvictionIter(self.eviction_iter())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
diff --git a/src/policies/mod.rs b/src/policies/mod.rs
index 9bb740b3..9938d990 100644
--- a/src/policies/mod.rs
+++ b/src/policies/mod.rs
@@ -20,11 +20,37 @@
 pub mod lru;
 pub mod tinylfu;
 
-use std::hash::Hash;
+use std::{hash::Hash, ptr::NonNull};
 
 pub trait Index: Clone + Hash + Send + Sync + 'static {}
 
-pub trait Policy: Send + Sync + 'static {}
+pub trait Policy: Send + Sync + 'static {
+    type I: Index;
+
+    fn add(&mut self, handle: NonNull<Handle<Self::I>>) -> bool;
+
+    fn remove(&mut self, handle: NonNull<Handle<Self::I>>) -> bool;
+
+    fn record_access(&mut self, handle: NonNull<Handle<Self::I>>, mode: AccessMode) -> bool;
+
+    fn eviction_iter(&mut self) -> EvictionIter<'_, Self::I>;
+}
+
+pub enum EvictionIter<'a, I: Index> {
+    LruEvictionIter(lru::EvictionIter<'a, I>),
+    TinyLfuEvictionIter(tinylfu::EvictionIter<'a, I>),
+}
+
+impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
+    type Item = &'a I;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            EvictionIter::LruEvictionIter(iter) => iter.next(),
+            EvictionIter::TinyLfuEvictionIter(iter) => iter.next(),
+        }
+    }
+}
 
 #[derive(PartialEq, Eq, Debug)]
 pub enum AccessMode {
@@ -32,14 +58,38 @@ pub enum AccessMode {
     Write,
 }
 
-pub enum HandleInner<I: Index> {
+pub enum Handle<I: Index> {
     LruHandle(lru::Handle<I>),
+    TinyLfuHandle(tinylfu::Handle<I>),
 }
 
-#[allow(unused)]
-pub struct Handle<I: Index> {
-    inner: HandleInner<I>,
+impl<I: Index> Handle<I> {
+    pub fn index(&self) -> &I {
+        match self {
+            Handle::LruHandle(handle) => handle.index(),
+            Handle::TinyLfuHandle(handle) => handle.index(),
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! extract_handle {
+    ($handle:expr, $type:tt) => {
+        paste::paste! {
+            unsafe {
+                let inner = match $handle.as_mut() {
+                    $crate::policies::Handle::[<$type Handle>](handle) => handle,
+                    _ => unreachable!(),
+                };
+                std::ptr::NonNull::new_unchecked(inner as *mut _)
+            }
+        }
+    };
 }
 
 #[cfg(test)]
-impl Index for u64 {}
+mod tests {
+    use super::*;
+
+    impl Index for u64 {}
+}
diff --git a/src/policies/tinylfu.rs b/src/policies/tinylfu.rs
index e0f8dc56..97931444 100644
--- a/src/policies/tinylfu.rs
+++ b/src/policies/tinylfu.rs
@@ -34,9 +34,9 @@ use cmsketch::CMSketchUsize;
 use twox_hash::XxHash64;
 
 use crate::collections::dlist::{DList, Entry, Iter};
-use crate::intrusive_dlist;
+use crate::{extract_handle, intrusive_dlist};
 
-use super::{AccessMode, Index};
+use super::{AccessMode, Index, Policy};
 
 const MIN_CAPACITY: usize = 100;
 const ERROR_THRESHOLD: f64 = 5.0;
@@ -89,6 +89,10 @@ impl<I: Index> Handle<I> {
             index,
         }
     }
+
+    pub fn index(&self) -> &I {
+        &self.index
+    }
 }
 
 intrusive_dlist! { Handle<I: Index>, entry_tiny, HandleDListTinyAdapter}
@@ -166,7 +170,7 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
     /// returns `false` otherwise.
-    pub fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
+    fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
         unsafe {
             if (mode == AccessMode::Read && !self.config.update_on_read)
                 || (mode == AccessMode::Write && !self.config.update_on_write)
@@ -192,7 +196,7 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if handle is successfully added into the lru,
     /// returns `false` if the handle is already in the lru.
-    pub fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
             if handle.as_ref().is_in_cache {
                 return false;
@@ -231,7 +235,7 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if handle is successfully removed from the lru,
     /// returns `false` if the handle is unchanged.
-    pub fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
             if !handle.as_ref().is_in_cache {
                 return false;
@@ -249,7 +253,7 @@ impl<I: Index> TinyLfu<I> {
         }
     }
 
-    pub fn eviction_iter<'a>(&'a mut self) -> EvictionIter<'a, I> {
+    fn eviction_iter<'a>(&'a mut self) -> EvictionIter<'a, I> {
         unsafe {
             let mut iter_main: Iter<'a, _, _> = self.lru_main.iter();
             iter_main.tail();
@@ -393,10 +397,37 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
     }
 }
 
-// unsafe impl `Send + Sync` for `Lru` because it uses `NonNull`
+// unsafe impl `Send + Sync` for `TinyLfu` because it uses `NonNull`
 unsafe impl<I: Index> Send for TinyLfu<I> {}
 unsafe impl<I: Index> Sync for TinyLfu<I> {}
 
+impl<I: Index> Policy for TinyLfu<I> {
+    type I = I;
+
+    fn add(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
+        let handle = extract_handle!(handle, TinyLfu);
+        self.add(handle)
+    }
+
+    fn remove(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
+        let handle = extract_handle!(handle, TinyLfu);
+        self.remove(handle)
+    }
+
+    fn record_access(
+        &mut self,
+        mut handle: NonNull<super::Handle<Self::I>>,
+        mode: AccessMode,
+    ) -> bool {
+        let handle = extract_handle!(handle, TinyLfu);
+        self.record_access(handle, mode)
+    }
+
+    fn eviction_iter(&mut self) -> super::EvictionIter<'_, Self::I> {
+        super::EvictionIter::TinyLfuEvictionIter(self.eviction_iter())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;

From cee5aaac4200f1bdbe16355b9fbec687396fbe42 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 22 May 2023 14:13:37 +0800
Subject: [PATCH 006/261] feat: introduce container framework (#7)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock              |   4 +-
 src/container.rs        | 255 ++++++++++++++++++++++++++++++++++++++++
 src/lib.rs              |  13 +-
 src/policies/lru.rs     |  86 ++++++++------
 src/policies/mod.rs     |  74 +++---------
 src/policies/tinylfu.rs |  88 ++++++++------
 src/store/mod.rs        |  70 +++++++++++
 7 files changed, 451 insertions(+), 139 deletions(-)
 create mode 100644 src/container.rs
 create mode 100644 src/store/mod.rs

diff --git a/Cargo.lock b/Cargo.lock
index 23466cba..299a7dd3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -22,9 +22,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cmsketch"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52c83ab4269fb46767c201bcf7957df0eb14988fbea81fef4efbc8ba8f0241ef"
+checksum = "041ac3f10d521f2049bf4363edc19b63db525b496ad422c99dd3158f4c2e5999"
 dependencies = [
  "paste",
 ]
diff --git a/src/container.rs b/src/container.rs
new file mode 100644
index 00000000..2ca57c20
--- /dev/null
+++ b/src/container.rs
@@ -0,0 +1,255 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::collections::BTreeMap;
+use std::hash::Hasher;
+use std::marker::PhantomData;
+use std::ptr::NonNull;
+
+use itertools::Itertools;
+use parking_lot::{Mutex, MutexGuard};
+use twox_hash::XxHash64;
+
+use crate::policies::{Handle, Policy};
+use crate::store::Store;
+use crate::{Data, Index};
+
+pub struct Config<I, P, H, T, W, S>
+where
+    I: Index,
+    P: Policy<I = I, H = H>,
+    H: Handle<I = I>,
+    W: Fn(&T) -> usize + Send + Sync,
+    S: Store<I = I, D = T>,
+{
+    capacity: usize,
+
+    pool_count_bits: usize,
+
+    policy_config: P::C,
+
+    weighter: W,
+
+    store: S,
+
+    _marker: PhantomData<(I, H, T)>,
+}
+
+#[allow(clippy::type_complexity)]
+pub struct Container<I, P, H, D, W, S>
+where
+    I: Index,
+    P: Policy<I = I, H = H>,
+    H: Handle<I = I>,
+    D: Data,
+    W: Fn(&D) -> usize + Send + Sync,
+    S: Store<I = I, D = D>,
+{
+    pool_count_bits: usize,
+    pools: Vec<Mutex<Pool<I, P, H, D, S>>>,
+
+    weighter: W,
+}
+
+impl<I, P, H, D, W, S> Container<I, P, H, D, W, S>
+where
+    I: Index,
+    P: Policy<I = I, H = H>,
+    H: Handle<I = I>,
+    D: Data,
+    W: Fn(&D) -> usize + Send + Sync,
+    S: Store<I = I, D = D>,
+{
+    pub fn new(config: Config<I, P, H, D, W, S>) -> Self {
+        let pool_count = 1 << config.pool_count_bits;
+        let capacity = config.capacity >> config.pool_count_bits;
+        let pools = (0..pool_count)
+            .map(|id| Pool {
+                id,
+                policy: P::new(config.policy_config.clone()),
+                capacity,
+                size: 0,
+                handles: BTreeMap::new(),
+                store: config.store.clone(),
+            })
+            .map(Mutex::new)
+            .collect_vec();
+
+        Self {
+            pool_count_bits: config.pool_count_bits,
+            pools,
+
+            weighter: config.weighter,
+        }
+    }
+
+    pub fn insert(&self, index: I, data: D) -> bool {
+        let mut pool = self.pool(&index);
+
+        if pool.handles.get(&index).is_some() {
+            // already in cache
+            return false;
+        }
+
+        let weight = (self.weighter)(&data);
+
+        pool.make_room(weight);
+
+        pool.insert(index, weight, data);
+
+        true
+    }
+
+    pub fn remove(&self, index: &I) -> bool {
+        let mut pool = self.pool(index);
+
+        if pool.handles.get(index).is_none() {
+            // not in cache
+            return false;
+        }
+
+        pool.remove(index);
+
+        true
+    }
+
+    pub fn get(&self, index: &I) -> Option<D> {
+        let mut pool = self.pool(index);
+
+        pool.get(index)
+    }
+
+    fn pool(&self, index: &I) -> MutexGuard<'_, Pool<I, P, H, D, S>> {
+        let mut hasher = XxHash64::default();
+        index.hash(&mut hasher);
+        let id = hasher.finish() as usize & ((1 << self.pool_count_bits) - 1);
+
+        self.pools[id].lock()
+    }
+}
+
+struct Pool<I, P, H, D, S>
+where
+    I: Index,
+    P: Policy<I = I, H = H>,
+    H: Handle<I = I>,
+    D: Data,
+    S: Store<I = I, D = D>,
+{
+    #[allow(unused)]
+    id: usize,
+
+    policy: P,
+
+    capacity: usize,
+
+    size: usize,
+
+    handles: BTreeMap<I, PoolHandle<I, H>>,
+
+    store: S,
+}
+
+impl<I, P, H, D, S> Pool<I, P, H, D, S>
+where
+    I: Index,
+    P: Policy<I = I, H = H>,
+    H: Handle<I = I>,
+    D: Data,
+    S: Store<I = I, D = D>,
+{
+    fn make_room(&mut self, weight: usize) {
+        let mut handles = vec![];
+        for index in self.policy.eviction_iter() {
+            if self.size + weight <= self.capacity || self.size == 0 {
+                break;
+            }
+            let PoolHandle { weight, handle } = self.handles.remove(index).unwrap();
+            self.size -= weight;
+            handles.push(handle);
+        }
+        for handle in handles {
+            assert!(self.policy.remove(handle));
+            unsafe { drop(Box::from_raw(handle.as_ptr())) };
+        }
+    }
+
+    fn insert(&mut self, index: I, weight: usize, data: D) {
+        let handle = Box::new(H::new(index.clone()));
+        let handle = unsafe { NonNull::new_unchecked(Box::leak(handle)) };
+
+        assert!(self.policy.insert(handle));
+        self.handles
+            .insert(index.clone(), PoolHandle { weight, handle });
+        self.size += weight;
+
+        self.store.store(index, data);
+    }
+
+    fn remove(&mut self, index: &I) {
+        let PoolHandle { weight, handle } = self.handles.remove(index).unwrap();
+        assert!(self.policy.remove(handle));
+        self.size -= weight;
+
+        self.store.delete(index);
+    }
+
+    fn get(&mut self, index: &I) -> Option<D> {
+        match self.handles.get(index) {
+            Some(PoolHandle { weight: _, handle }) => {
+                self.policy.access(*handle);
+                self.store.load(index)
+            }
+            None => None,
+        }
+    }
+}
+
+struct PoolHandle<I, H>
+where
+    I: Index,
+    H: Handle<I = I>,
+{
+    weight: usize,
+    handle: NonNull<H>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::policies::lru::{Config as LruConfig, Handle as LruHandle, Lru};
+    use crate::store::tests::MemoryStore;
+
+    #[test]
+    fn test() {
+        let policy_config = LruConfig {
+            update_on_write: true,
+            update_on_read: true,
+            lru_insertion_point_fraction: 0.0,
+        };
+
+        let config = Config {
+            capacity: 1024 * 1024 * 4,
+            pool_count_bits: 5,
+            policy_config,
+            weighter: |data: &Vec<u8>| data.len(),
+            store: MemoryStore::new(),
+            _marker: PhantomData,
+        };
+
+        let _container: Container<u64, Lru<_>, LruHandle<_>, Vec<u8>, _, MemoryStore<_, _>> =
+            Container::new(config);
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index e7263c75..08aa7f96 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,12 +12,15 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use policies::Policy;
+#![feature(trait_alias)]
+
+pub use policies::Policy;
 
 pub mod collections;
+pub mod container;
 pub mod policies;
+pub mod store;
 
-#[allow(unused)]
-pub struct Cache<P: Policy> {
-    policy: P,
-}
+pub trait Index =
+    PartialOrd + Ord + PartialEq + Eq + Clone + std::hash::Hash + Send + Sync + 'static;
+pub trait Data = Send + Sync + 'static;
diff --git a/src/policies/lru.rs b/src/policies/lru.rs
index b0f96b06..374d3f91 100644
--- a/src/policies/lru.rs
+++ b/src/policies/lru.rs
@@ -30,17 +30,18 @@ use std::ptr::NonNull;
 use std::time::SystemTime;
 
 use crate::collections::dlist::{DList, Entry, Iter};
-use crate::{extract_handle, intrusive_dlist};
+use crate::intrusive_dlist;
 
-use super::{AccessMode, Index, Policy};
+use super::Index;
 
+#[derive(Clone, Debug)]
 pub struct Config {
-    update_on_write: bool,
+    pub update_on_write: bool,
 
-    update_on_read: bool,
+    pub update_on_read: bool,
 
     /// Insertion point of the new entry, between 0 and 1.
-    lru_insertion_point_fraction: f64,
+    pub lru_insertion_point_fraction: f64,
 }
 
 pub struct Handle<I: Index> {
@@ -56,7 +57,7 @@ pub struct Handle<I: Index> {
 }
 
 impl<I: Index> Handle<I> {
-    pub fn new(index: I) -> Self {
+    fn new(index: I) -> Self {
         Self {
             entry: Entry::default(),
 
@@ -70,7 +71,7 @@ impl<I: Index> Handle<I> {
         }
     }
 
-    pub fn index(&self) -> &I {
+    fn index(&self) -> &I {
         &self.index
     }
 }
@@ -91,7 +92,7 @@ pub struct Lru<I: Index> {
 }
 
 impl<I: Index> Lru<I> {
-    pub fn new(config: Config) -> Self {
+    fn new(config: Config) -> Self {
         Self {
             lru: DList::new(),
 
@@ -105,13 +106,8 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
     /// returns `false` otherwise.
-    fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
+    fn access(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
-            if (mode == AccessMode::Read && !self.config.update_on_read)
-                || (mode == AccessMode::Write && !self.config.update_on_write)
-            {
-                return false;
-            }
             handle.as_mut().is_accessed = true;
 
             // TODO(MrCroxx): try trigger reconfigure
@@ -135,7 +131,7 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if handle is successfully added into the lru,
     /// returns `false` if the handle is already in the lru.
-    fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn insert(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
             if handle.as_ref().is_in_cache {
                 return false;
@@ -174,7 +170,7 @@ impl<I: Index> Lru<I> {
         }
     }
 
-    fn eviction_iter(&mut self) -> EvictionIter<'_, I> {
+    fn eviction_iter(&self) -> EvictionIter<'_, I> {
         unsafe {
             let mut iter = self.lru.iter();
             iter.tail();
@@ -259,34 +255,52 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
     }
 }
 
-// unsafe impl `Send + Sync` for `Lru` because it uses `NonNull`
+// unsafe impl `Send + Sync` for structs with `NonNull` usage
+
 unsafe impl<I: Index> Send for Lru<I> {}
 unsafe impl<I: Index> Sync for Lru<I> {}
 
-impl<I: Index> Policy for Lru<I> {
+unsafe impl<I: Index> Send for Handle<I> {}
+unsafe impl<I: Index> Sync for Handle<I> {}
+
+impl super::Config for Config {}
+
+impl<I: Index> super::Handle for Handle<I> {
+    type I = I;
+
+    fn new(index: Self::I) -> Self {
+        Self::new(index)
+    }
+
+    fn index(&self) -> &Self::I {
+        self.index()
+    }
+}
+
+impl<I: Index> super::Policy for Lru<I> {
     type I = I;
+    type C = Config;
+    type H = Handle<I>;
+    type E<'e> = EvictionIter<'e, I>;
+
+    fn new(config: Self::C) -> Self {
+        Lru::new(config)
+    }
 
-    fn add(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
-        let handle = extract_handle!(handle, Lru);
-        self.add(handle)
+    fn insert(&mut self, handle: NonNull<Self::H>) -> bool {
+        self.insert(handle)
     }
 
-    fn remove(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
-        let handle = extract_handle!(handle, Lru);
+    fn remove(&mut self, handle: NonNull<Self::H>) -> bool {
         self.remove(handle)
     }
 
-    fn record_access(
-        &mut self,
-        mut handle: NonNull<super::Handle<Self::I>>,
-        mode: AccessMode,
-    ) -> bool {
-        let handle = extract_handle!(handle, Lru);
-        self.record_access(handle, mode)
+    fn access(&mut self, handle: NonNull<Self::H>) -> bool {
+        self.access(handle)
     }
 
-    fn eviction_iter(&mut self) -> super::EvictionIter<'_, Self::I> {
-        super::EvictionIter::LruEvictionIter(self.eviction_iter())
+    fn eviction_iter(&self) -> Self::E<'_> {
+        self.eviction_iter()
     }
 }
 
@@ -311,13 +325,13 @@ mod tests {
 
         let mut handles = vec![Handle::new(0), Handle::new(1), Handle::new(2)];
 
-        lru.add(ptr(&mut handles[0]));
-        lru.add(ptr(&mut handles[1]));
-        lru.add(ptr(&mut handles[2]));
+        lru.insert(ptr(&mut handles[0]));
+        lru.insert(ptr(&mut handles[1]));
+        lru.insert(ptr(&mut handles[2]));
 
         assert_eq!(vec![0, 1, 2], lru.eviction_iter().copied().collect_vec());
 
-        lru.record_access(ptr(&mut handles[1]), AccessMode::Read);
+        lru.access(ptr(&mut handles[1]));
 
         assert_eq!(vec![0, 2, 1], lru.eviction_iter().copied().collect_vec());
 
diff --git a/src/policies/mod.rs b/src/policies/mod.rs
index 9938d990..06d224f1 100644
--- a/src/policies/mod.rs
+++ b/src/policies/mod.rs
@@ -20,76 +20,32 @@
 pub mod lru;
 pub mod tinylfu;
 
-use std::{hash::Hash, ptr::NonNull};
-
-pub trait Index: Clone + Hash + Send + Sync + 'static {}
+use crate::Index;
+use std::ptr::NonNull;
 
 pub trait Policy: Send + Sync + 'static {
     type I: Index;
+    type C: Config;
+    type H: Handle<I = Self::I>;
+    type E<'e>: Iterator<Item = &'e Self::I>;
 
-    fn add(&mut self, handle: NonNull<Handle<Self::I>>) -> bool;
-
-    fn remove(&mut self, handle: NonNull<Handle<Self::I>>) -> bool;
-
-    fn record_access(&mut self, handle: NonNull<Handle<Self::I>>, mode: AccessMode) -> bool;
-
-    fn eviction_iter(&mut self) -> EvictionIter<'_, Self::I>;
-}
-
-pub enum EvictionIter<'a, I: Index> {
-    LruEvictionIter(lru::EvictionIter<'a, I>),
-    TinyLfuEvictionIter(tinylfu::EvictionIter<'a, I>),
-}
+    fn new(config: Self::C) -> Self;
 
-impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
-    type Item = &'a I;
+    fn insert(&mut self, handle: NonNull<Self::H>) -> bool;
 
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            EvictionIter::LruEvictionIter(iter) => iter.next(),
-            EvictionIter::TinyLfuEvictionIter(iter) => iter.next(),
-        }
-    }
-}
+    fn remove(&mut self, handle: NonNull<Self::H>) -> bool;
 
-#[derive(PartialEq, Eq, Debug)]
-pub enum AccessMode {
-    Read,
-    Write,
-}
+    fn access(&mut self, handle: NonNull<Self::H>) -> bool;
 
-pub enum Handle<I: Index> {
-    LruHandle(lru::Handle<I>),
-    TinyLfuHandle(tinylfu::Handle<I>),
+    fn eviction_iter(&self) -> Self::E<'_>;
 }
 
-impl<I: Index> Handle<I> {
-    pub fn index(&self) -> &I {
-        match self {
-            Handle::LruHandle(handle) => handle.index(),
-            Handle::TinyLfuHandle(handle) => handle.index(),
-        }
-    }
-}
+pub trait Config: Send + Sync + Clone + 'static {}
 
-#[macro_export]
-macro_rules! extract_handle {
-    ($handle:expr, $type:tt) => {
-        paste::paste! {
-            unsafe {
-                let inner = match $handle.as_mut() {
-                    $crate::policies::Handle::[<$type Handle>](handle) => handle,
-                    _ => unreachable!(),
-                };
-                std::ptr::NonNull::new_unchecked(inner as *mut _)
-            }
-        }
-    };
-}
+pub trait Handle: Send + Sync + 'static {
+    type I: Index;
 
-#[cfg(test)]
-mod tests {
-    use super::*;
+    fn new(index: Self::I) -> Self;
 
-    impl Index for u64 {}
+    fn index(&self) -> &Self::I;
 }
diff --git a/src/policies/tinylfu.rs b/src/policies/tinylfu.rs
index 97931444..d690ecba 100644
--- a/src/policies/tinylfu.rs
+++ b/src/policies/tinylfu.rs
@@ -34,25 +34,26 @@ use cmsketch::CMSketchUsize;
 use twox_hash::XxHash64;
 
 use crate::collections::dlist::{DList, Entry, Iter};
-use crate::{extract_handle, intrusive_dlist};
+use crate::intrusive_dlist;
 
-use super::{AccessMode, Index, Policy};
+use super::Index;
 
 const MIN_CAPACITY: usize = 100;
 const ERROR_THRESHOLD: f64 = 5.0;
 const HASH_COUNT: usize = 4;
 const DECAY_FACTOR: f64 = 0.5;
 
+#[derive(Clone, Debug)]
 pub struct Config {
-    update_on_write: bool,
+    pub update_on_write: bool,
 
-    update_on_read: bool,
+    pub update_on_read: bool,
 
     /// The multiplier for window len given the cache size.
-    window_to_cache_size_ratio: usize,
+    pub window_to_cache_size_ratio: usize,
 
     /// The ratio of tiny lru capacity to overall capacity.
-    tiny_lru_capacity_ratio: f64,
+    pub tiny_lru_capacity_ratio: f64,
 }
 
 #[derive(PartialEq, Eq, Debug)]
@@ -75,7 +76,7 @@ pub struct Handle<I: Index> {
 }
 
 impl<I: Index> Handle<I> {
-    pub fn new(index: I) -> Self {
+    fn new(index: I) -> Self {
         Self {
             entry_tiny: Entry::default(),
             entry_main: Entry::default(),
@@ -90,7 +91,7 @@ impl<I: Index> Handle<I> {
         }
     }
 
-    pub fn index(&self) -> &I {
+    fn index(&self) -> &I {
         &self.index
     }
 }
@@ -170,13 +171,8 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
     /// returns `false` otherwise.
-    fn record_access(&mut self, mut handle: NonNull<Handle<I>>, mode: AccessMode) -> bool {
+    fn access(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
-            if (mode == AccessMode::Read && !self.config.update_on_read)
-                || (mode == AccessMode::Write && !self.config.update_on_write)
-            {
-                return false;
-            }
             handle.as_mut().is_accessed = true;
 
             if !handle.as_ref().is_in_cache {
@@ -196,7 +192,7 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if handle is successfully added into the lru,
     /// returns `false` if the handle is already in the lru.
-    fn add(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn insert(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
         unsafe {
             if handle.as_ref().is_in_cache {
                 return false;
@@ -253,7 +249,7 @@ impl<I: Index> TinyLfu<I> {
         }
     }
 
-    fn eviction_iter<'a>(&'a mut self) -> EvictionIter<'a, I> {
+    fn eviction_iter<'a>(&'a self) -> EvictionIter<'a, I> {
         unsafe {
             let mut iter_main: Iter<'a, _, _> = self.lru_main.iter();
             iter_main.tail();
@@ -292,7 +288,7 @@ impl<I: Index> TinyLfu<I> {
     }
 
     fn update_frequencies(&mut self, handle: NonNull<Handle<I>>) {
-        self.frequencies.add(Self::hash_handle(handle));
+        self.frequencies.record(Self::hash_handle(handle));
         self.window_size += 1;
 
         // Decay counts every `max_window_size`. This avoids having items that were
@@ -397,34 +393,52 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
     }
 }
 
-// unsafe impl `Send + Sync` for `TinyLfu` because it uses `NonNull`
+// unsafe impl `Send + Sync` for structs with `NonNull` usage
+
 unsafe impl<I: Index> Send for TinyLfu<I> {}
 unsafe impl<I: Index> Sync for TinyLfu<I> {}
 
-impl<I: Index> Policy for TinyLfu<I> {
+unsafe impl<I: Index> Send for Handle<I> {}
+unsafe impl<I: Index> Sync for Handle<I> {}
+
+impl super::Config for Config {}
+
+impl<I: Index> super::Handle for Handle<I> {
     type I = I;
 
-    fn add(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
-        let handle = extract_handle!(handle, TinyLfu);
-        self.add(handle)
+    fn new(index: Self::I) -> Self {
+        Self::new(index)
+    }
+
+    fn index(&self) -> &Self::I {
+        self.index()
     }
+}
 
-    fn remove(&mut self, mut handle: NonNull<super::Handle<Self::I>>) -> bool {
-        let handle = extract_handle!(handle, TinyLfu);
+impl<I: Index> super::Policy for TinyLfu<I> {
+    type I = I;
+    type C = Config;
+    type H = Handle<I>;
+    type E<'e> = EvictionIter<'e, I>;
+
+    fn new(config: Self::C) -> Self {
+        TinyLfu::new(config)
+    }
+
+    fn insert(&mut self, handle: NonNull<Self::H>) -> bool {
+        self.insert(handle)
+    }
+
+    fn remove(&mut self, handle: NonNull<Self::H>) -> bool {
         self.remove(handle)
     }
 
-    fn record_access(
-        &mut self,
-        mut handle: NonNull<super::Handle<Self::I>>,
-        mode: AccessMode,
-    ) -> bool {
-        let handle = extract_handle!(handle, TinyLfu);
-        self.record_access(handle, mode)
+    fn access(&mut self, handle: NonNull<Self::H>) -> bool {
+        self.access(handle)
     }
 
-    fn eviction_iter(&mut self) -> super::EvictionIter<'_, Self::I> {
-        super::EvictionIter::TinyLfuEvictionIter(self.eviction_iter())
+    fn eviction_iter(&self) -> Self::E<'_> {
+        self.eviction_iter()
     }
 }
 
@@ -450,7 +464,7 @@ mod tests {
 
         let mut handles = (0..101).map(Handle::new).collect_vec();
         for handle in handles.iter_mut().take(100) {
-            lfu.add(ptr(handle));
+            lfu.insert(ptr(handle));
         }
 
         assert_eq!(99, lfu.lru_main.len());
@@ -464,10 +478,10 @@ mod tests {
         );
 
         for handle in handles.iter_mut().take(100) {
-            lfu.record_access(ptr(handle), AccessMode::Read);
+            lfu.access(ptr(handle));
         }
-        lfu.record_access(ptr(&mut handles[0]), AccessMode::Read);
-        lfu.add(ptr(&mut handles[100]));
+        lfu.access(ptr(&mut handles[0]));
+        lfu.insert(ptr(&mut handles[100]));
 
         assert_eq!(handles[0].lru_type, LruType::Main);
         assert_eq!(handles[100].lru_type, LruType::Tiny);
diff --git a/src/store/mod.rs b/src/store/mod.rs
new file mode 100644
index 00000000..eed38f33
--- /dev/null
+++ b/src/store/mod.rs
@@ -0,0 +1,70 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use crate::{Data, Index};
+
+pub trait Store: Send + Sync + Clone + 'static {
+    type I: Index;
+    type D: Data;
+
+    fn store(&self, index: Self::I, data: Self::D);
+
+    fn load(&self, index: &Self::I) -> Option<Self::D>;
+
+    fn delete(&self, index: &Self::I);
+}
+
+#[cfg(test)]
+pub mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use parking_lot::RwLock;
+
+    use super::*;
+
+    #[derive(Clone, Debug, Default)]
+    pub struct MemoryStore<I: Index, D: Data + Clone> {
+        inner: Arc<RwLock<HashMap<I, D>>>,
+    }
+
+    impl<I: Index, D: Data + Clone> MemoryStore<I, D> {
+        pub fn new() -> Self {
+            Self {
+                inner: Arc::new(RwLock::new(HashMap::default())),
+            }
+        }
+    }
+
+    impl<I: Index, D: Data + Clone> Store for MemoryStore<I, D> {
+        type I = I;
+
+        type D = D;
+
+        fn store(&self, index: Self::I, data: Self::D) {
+            let mut inner = self.inner.write();
+            inner.insert(index, data);
+        }
+
+        fn load(&self, index: &Self::I) -> Option<Self::D> {
+            let inner = self.inner.read();
+            inner.get(index).cloned()
+        }
+
+        fn delete(&self, index: &Self::I) {
+            let mut inner = self.inner.write();
+            inner.remove(index);
+        }
+    }
+}

From 3992c3f02caff20ca3a90792b231005147f10312 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 22 May 2023 14:31:29 +0800
Subject: [PATCH 007/261] test: add simple test for container (#8)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 src/container.rs | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/container.rs b/src/container.rs
index 2ca57c20..b46b5a1f 100644
--- a/src/container.rs
+++ b/src/container.rs
@@ -130,6 +130,11 @@ where
         pool.get(index)
     }
 
+    // TODO(MrCroxx): optimize this
+    pub fn size(&self) -> usize {
+        self.pools.iter().map(|pool| pool.lock().size).sum()
+    }
+
     fn pool(&self, index: &I) -> MutexGuard<'_, Pool<I, P, H, D, S>> {
         let mut hasher = XxHash64::default();
         index.hash(&mut hasher);
@@ -233,7 +238,7 @@ mod tests {
     use crate::store::tests::MemoryStore;
 
     #[test]
-    fn test() {
+    fn test_container_simple() {
         let policy_config = LruConfig {
             update_on_write: true,
             update_on_read: true,
@@ -241,15 +246,32 @@ mod tests {
         };
 
         let config = Config {
-            capacity: 1024 * 1024 * 4,
-            pool_count_bits: 5,
+            capacity: 100,
+            pool_count_bits: 0,
             policy_config,
             weighter: |data: &Vec<u8>| data.len(),
             store: MemoryStore::new(),
             _marker: PhantomData,
         };
 
-        let _container: Container<u64, Lru<_>, LruHandle<_>, Vec<u8>, _, MemoryStore<_, _>> =
+        let container: Container<u64, Lru<_>, LruHandle<_>, Vec<u8>, _, MemoryStore<_, _>> =
             Container::new(config);
+
+        assert!(container.insert(1, vec![b'x'; 40]));
+        assert!(!container.insert(1, vec![b'x'; 40]));
+        assert_eq!(container.get(&1), Some(vec![b'x'; 40]));
+
+        assert!(container.insert(2, vec![b'x'; 60]));
+        assert!(!container.insert(2, vec![b'x'; 60]));
+        assert_eq!(container.get(&2), Some(vec![b'x'; 60]));
+
+        assert!(container.insert(3, vec![b'x'; 50]));
+        assert_eq!(container.get(&3), Some(vec![b'x'; 50]));
+        assert_eq!(container.get(&1), None);
+        assert_eq!(container.get(&2), None);
+
+        assert!(container.remove(&3));
+
+        assert_eq!(container.size(), 0);
     }
 }

From 24cccaf192a67c98c9ccf0a983989f3fa65a26fe Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 24 May 2023 15:31:48 +0800
Subject: [PATCH 008/261] feat: introduce read only file store (#9)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock                        | 389 +++++++++++++++++++++++++++--
 Cargo.toml                        |  17 +-
 src/collections/dlist.rs          |   3 +-
 src/container.rs                  | 124 +++++----
 src/lib.rs                        |  38 ++-
 src/policies/lru.rs               |   3 +-
 src/policies/tinylfu.rs           |   3 +-
 src/store/error.rs                |  29 +++
 src/store/file.rs                 | 400 ++++++++++++++++++++++++++++++
 src/store/mod.rs                  |  59 ++++-
 src/store/read_only_file_store.rs | 377 ++++++++++++++++++++++++++++
 src/store/utils.rs                | 111 +++++++++
 12 files changed, 1467 insertions(+), 86 deletions(-)
 create mode 100644 src/store/error.rs
 create mode 100644 src/store/file.rs
 create mode 100644 src/store/read_only_file_store.rs
 create mode 100644 src/store/utils.rs

diff --git a/Cargo.lock b/Cargo.lock
index 299a7dd3..df8f2062 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,17 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "async-trait"
+version = "0.1.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -14,6 +25,18 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bytes"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+
+[[package]]
+name = "cc"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -22,13 +45,48 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cmsketch"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "041ac3f10d521f2049bf4363edc19b63db525b496ad422c99dd3158f4c2e5999"
+checksum = "467e460587e81453bf9aeb43cd534e9c5ad670042023bd6c3f377c23b76cc2f0"
 dependencies = [
  "paste",
 ]
 
+[[package]]
+name = "crossbeam"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c"
+dependencies = [
+ "cfg-if",
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
+ "crossbeam-queue",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-epoch"
 version = "0.9.14"
@@ -38,10 +96,20 @@ dependencies = [
  "autocfg",
  "cfg-if",
  "crossbeam-utils",
- "memoffset",
+ "memoffset 0.8.0",
  "scopeguard",
 ]
 
+[[package]]
+name = "crossbeam-queue"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.15"
@@ -57,19 +125,55 @@ version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
 
+[[package]]
+name = "errno"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "fastrand"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
+dependencies = [
+ "instant",
+]
+
 [[package]]
 name = "foyer"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
+ "bytes",
  "cmsketch",
- "crossbeam-epoch",
- "crossbeam-utils",
+ "crossbeam",
  "futures",
  "itertools",
- "memoffset",
+ "libc",
+ "memoffset 0.8.0",
+ "nix",
  "parking_lot",
  "paste",
  "rand_mt",
+ "tempfile",
+ "thiserror",
+ "tokio",
  "twox-hash",
 ]
 
@@ -173,6 +277,41 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "io-lifetimes"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
+dependencies = [
+ "hermit-abi 0.3.1",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -188,6 +327,12 @@ version = "0.2.144"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
+
 [[package]]
 name = "lock_api"
 version = "0.4.9"
@@ -198,12 +343,30 @@ dependencies = [
  "scopeguard",
 ]
 
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.8.0"
@@ -213,6 +376,42 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "mio"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
+dependencies = [
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys 0.45.0",
+]
+
+[[package]]
+name = "nix"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "libc",
+ "memoffset 0.7.1",
+ "pin-utils",
+ "static_assertions",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
+dependencies = [
+ "hermit-abi 0.2.6",
+ "libc",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -231,9 +430,9 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.2.16",
  "smallvec",
- "windows-sys",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -326,12 +525,44 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "rustix"
+version = "0.37.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.8"
@@ -364,6 +595,66 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "redox_syscall 0.3.5",
+ "rustix",
+ "windows-sys 0.45.0",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tokio"
+version = "1.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
+dependencies = [
+ "autocfg",
+ "libc",
+ "mio",
+ "num_cpus",
+ "pin-project-lite",
+ "signal-hook-registry",
+ "tokio-macros",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "twox-hash"
 version = "1.6.3"
@@ -393,7 +684,16 @@ version = "0.45.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.42.2",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.0",
 ]
 
 [[package]]
@@ -402,13 +702,28 @@ version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.0",
+ "windows_aarch64_msvc 0.48.0",
+ "windows_i686_gnu 0.48.0",
+ "windows_i686_msvc 0.48.0",
+ "windows_x86_64_gnu 0.48.0",
+ "windows_x86_64_gnullvm 0.48.0",
+ "windows_x86_64_msvc 0.48.0",
 ]
 
 [[package]]
@@ -417,38 +732,80 @@ version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
diff --git a/Cargo.toml b/Cargo.toml
index 848664f6..7ddea64c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,15 +8,28 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-trait = "0.1"
+bytes = "1"
 cmsketch = "0.1"
-crossbeam-epoch = "0.9"
+crossbeam = "0.8"
 futures = "0.3"
 itertools = "0.10.5"
+libc = "0.2"
 memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
 twox-hash = "1"
 
 [dev-dependencies]
-crossbeam-utils = "0.8"
 rand_mt = "4.2.1"
+tempfile = "3"
diff --git a/src/collections/dlist.rs b/src/collections/dlist.rs
index a87b46ff..4e5a2ce8 100644
--- a/src/collections/dlist.rs
+++ b/src/collections/dlist.rs
@@ -33,7 +33,6 @@ pub trait Adapter<E> {
     fn el2en(_: NonNull<E>) -> NonNull<Entry>;
 }
 
-#[macro_export]
 macro_rules! intrusive_dlist {
     (
         $element:ident $(< $( $lt:tt $( : $clt:tt $(+ $dlt:tt)* )? ),+ >)?, $entry:ident, $adapter:ident
@@ -62,6 +61,8 @@ macro_rules! intrusive_dlist {
     };
 }
 
+pub(crate) use intrusive_dlist;
+
 /// TODO: write docs
 #[derive(Debug)]
 pub struct DList<E, A: Adapter<E>> {
diff --git a/src/container.rs b/src/container.rs
index b46b5a1f..caa9aefb 100644
--- a/src/container.rs
+++ b/src/container.rs
@@ -17,14 +17,18 @@ use std::hash::Hasher;
 use std::marker::PhantomData;
 use std::ptr::NonNull;
 
+use futures::future::try_join_all;
 use itertools::Itertools;
-use parking_lot::{Mutex, MutexGuard};
+use tokio::sync::{Mutex, MutexGuard};
 use twox_hash::XxHash64;
 
 use crate::policies::{Handle, Policy};
 use crate::store::Store;
 use crate::{Data, Index};
 
+// TODO(MrCroxx): wrap own result type
+use crate::store::error::Result;
+
 pub struct Config<I, P, H, T, W, S>
 where
     I: Index,
@@ -39,9 +43,9 @@ where
 
     policy_config: P::C,
 
-    weighter: W,
+    store_config: S::C,
 
-    store: S,
+    weighter: W,
 
     _marker: PhantomData<(I, H, T)>,
 }
@@ -71,76 +75,88 @@ where
     W: Fn(&D) -> usize + Send + Sync,
     S: Store<I = I, D = D>,
 {
-    pub fn new(config: Config<I, P, H, D, W, S>) -> Self {
+    pub async fn open(config: Config<I, P, H, D, W, S>) -> Result<Self> {
         let pool_count = 1 << config.pool_count_bits;
         let capacity = config.capacity >> config.pool_count_bits;
-        let pools = (0..pool_count)
-            .map(|id| Pool {
+
+        let stores = (0..pool_count)
+            .map(|pool| S::open(pool, config.store_config.clone()))
+            .collect_vec();
+        let stores = try_join_all(stores).await?;
+
+        let pools = stores
+            .into_iter()
+            .enumerate()
+            .map(|(id, store)| Pool {
                 id,
                 policy: P::new(config.policy_config.clone()),
                 capacity,
                 size: 0,
                 handles: BTreeMap::new(),
-                store: config.store.clone(),
+                store,
             })
             .map(Mutex::new)
             .collect_vec();
 
-        Self {
+        Ok(Self {
             pool_count_bits: config.pool_count_bits,
             pools,
 
             weighter: config.weighter,
-        }
+        })
     }
 
-    pub fn insert(&self, index: I, data: D) -> bool {
-        let mut pool = self.pool(&index);
+    pub async fn insert(&self, index: I, data: D) -> Result<bool> {
+        let mut pool = self.pool(&index).await;
 
         if pool.handles.get(&index).is_some() {
             // already in cache
-            return false;
+            return Ok(false);
         }
 
         let weight = (self.weighter)(&data);
 
-        pool.make_room(weight);
+        pool.make_room(weight).await?;
 
-        pool.insert(index, weight, data);
+        pool.insert(index, weight, data).await?;
 
-        true
+        Ok(true)
     }
 
-    pub fn remove(&self, index: &I) -> bool {
-        let mut pool = self.pool(index);
+    pub async fn remove(&self, index: &I) -> Result<bool> {
+        let mut pool = self.pool(index).await;
 
         if pool.handles.get(index).is_none() {
             // not in cache
-            return false;
+            return Ok(false);
         }
 
-        pool.remove(index);
+        pool.remove(index).await?;
 
-        true
+        Ok(true)
     }
 
-    pub fn get(&self, index: &I) -> Option<D> {
-        let mut pool = self.pool(index);
+    pub async fn get(&self, index: &I) -> Result<Option<D>> {
+        let mut pool = self.pool(index).await;
 
-        pool.get(index)
+        pool.get(index).await
     }
 
     // TODO(MrCroxx): optimize this
-    pub fn size(&self) -> usize {
-        self.pools.iter().map(|pool| pool.lock().size).sum()
+    pub async fn size(&self) -> usize {
+        let mut size = 0;
+        for pool in &self.pools {
+            size += pool.lock().await.size
+        }
+        size
     }
 
-    fn pool(&self, index: &I) -> MutexGuard<'_, Pool<I, P, H, D, S>> {
+    async fn pool(&self, index: &I) -> MutexGuard<'_, Pool<I, P, H, D, S>> {
         let mut hasher = XxHash64::default();
         index.hash(&mut hasher);
         let id = hasher.finish() as usize & ((1 << self.pool_count_bits) - 1);
 
-        self.pools[id].lock()
+        self.pools[id].lock().await
     }
 }
 
@@ -174,7 +190,7 @@ where
     D: Data,
     S: Store<I = I, D = D>,
 {
-    fn make_room(&mut self, weight: usize) {
+    async fn make_room(&mut self, weight: usize) -> Result<()> {
         let mut handles = vec![];
         for index in self.policy.eviction_iter() {
             if self.size + weight <= self.capacity || self.size == 0 {
@@ -183,14 +199,16 @@ where
             let PoolHandle { weight, handle } = self.handles.remove(index).unwrap();
             self.size -= weight;
             handles.push(handle);
+            self.store.delete(index).await?;
         }
         for handle in handles {
             assert!(self.policy.remove(handle));
             unsafe { drop(Box::from_raw(handle.as_ptr())) };
         }
+        Ok(())
     }
 
-    fn insert(&mut self, index: I, weight: usize, data: D) {
+    async fn insert(&mut self, index: I, weight: usize, data: D) -> Result<()> {
         let handle = Box::new(H::new(index.clone()));
         let handle = unsafe { NonNull::new_unchecked(Box::leak(handle)) };
 
@@ -199,24 +217,28 @@ where
             .insert(index.clone(), PoolHandle { weight, handle });
         self.size += weight;
 
-        self.store.store(index, data);
+        self.store.store(index, data).await?;
+
+        Ok(())
     }
 
-    fn remove(&mut self, index: &I) {
+    async fn remove(&mut self, index: &I) -> Result<()> {
         let PoolHandle { weight, handle } = self.handles.remove(index).unwrap();
         assert!(self.policy.remove(handle));
         self.size -= weight;
 
-        self.store.delete(index);
+        self.store.delete(index).await?;
+
+        Ok(())
     }
 
-    fn get(&mut self, index: &I) -> Option<D> {
+    async fn get(&mut self, index: &I) -> Result<Option<D>> {
         match self.handles.get(index) {
             Some(PoolHandle { weight: _, handle }) => {
                 self.policy.access(*handle);
-                self.store.load(index)
+                self.store.load(index).await
             }
-            None => None,
+            None => Ok(None),
         }
     }
 }
@@ -237,8 +259,8 @@ mod tests {
     use crate::policies::lru::{Config as LruConfig, Handle as LruHandle, Lru};
     use crate::store::tests::MemoryStore;
 
-    #[test]
-    fn test_container_simple() {
+    #[tokio::test]
+    async fn test_container_simple() {
         let policy_config = LruConfig {
             update_on_write: true,
             update_on_read: true,
@@ -250,28 +272,28 @@ mod tests {
             pool_count_bits: 0,
             policy_config,
             weighter: |data: &Vec<u8>| data.len(),
-            store: MemoryStore::new(),
+            store_config: (),
             _marker: PhantomData,
         };
 
         let container: Container<u64, Lru<_>, LruHandle<_>, Vec<u8>, _, MemoryStore<_, _>> =
-            Container::new(config);
+            Container::open(config).await.unwrap();
 
-        assert!(container.insert(1, vec![b'x'; 40]));
-        assert!(!container.insert(1, vec![b'x'; 40]));
-        assert_eq!(container.get(&1), Some(vec![b'x'; 40]));
+        assert!(container.insert(1, vec![b'x'; 40]).await.unwrap());
+        assert!(!container.insert(1, vec![b'x'; 40]).await.unwrap());
+        assert_eq!(container.get(&1).await.unwrap(), Some(vec![b'x'; 40]));
 
-        assert!(container.insert(2, vec![b'x'; 60]));
-        assert!(!container.insert(2, vec![b'x'; 60]));
-        assert_eq!(container.get(&2), Some(vec![b'x'; 60]));
+        assert!(container.insert(2, vec![b'x'; 60]).await.unwrap());
+        assert!(!container.insert(2, vec![b'x'; 60]).await.unwrap());
+        assert_eq!(container.get(&2).await.unwrap(), Some(vec![b'x'; 60]));
 
-        assert!(container.insert(3, vec![b'x'; 50]));
-        assert_eq!(container.get(&3), Some(vec![b'x'; 50]));
-        assert_eq!(container.get(&1), None);
-        assert_eq!(container.get(&2), None);
+        assert!(container.insert(3, vec![b'x'; 50]).await.unwrap());
+        assert_eq!(container.get(&3).await.unwrap(), Some(vec![b'x'; 50]));
+        assert_eq!(container.get(&1).await.unwrap(), None);
+        assert_eq!(container.get(&2).await.unwrap(), None);
 
-        assert!(container.remove(&3));
+        assert!(container.remove(&3).await.unwrap());
 
-        assert_eq!(container.size(), 0);
+        assert_eq!(container.size().await, 0);
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 08aa7f96..e84b421d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,6 +13,7 @@
 //  limitations under the License.
 
 #![feature(trait_alias)]
+#![feature(pattern)]
 
 pub use policies::Policy;
 
@@ -21,6 +22,37 @@ pub mod container;
 pub mod policies;
 pub mod store;
 
-pub trait Index =
-    PartialOrd + Ord + PartialEq + Eq + Clone + std::hash::Hash + Send + Sync + 'static;
-pub trait Data = Send + Sync + 'static;
+pub trait Index:
+    PartialOrd + Ord + PartialEq + Eq + Clone + std::hash::Hash + Send + Sync + 'static
+{
+    fn size() -> usize;
+
+    fn write(&self, buf: &mut [u8]);
+
+    fn read(buf: &[u8]) -> Self;
+}
+pub trait Data = Send + Sync + 'static + Into<Vec<u8>> + From<Vec<u8>>;
+
+#[cfg(test)]
+
+pub mod tests {
+    use bytes::{Buf, BufMut};
+
+    use super::*;
+
+    pub fn is_send_sync_static<T: Send + Sync + 'static>() {}
+
+    impl Index for u64 {
+        fn size() -> usize {
+            8
+        }
+
+        fn write(&self, mut buf: &mut [u8]) {
+            buf.put_u64(*self)
+        }
+
+        fn read(mut buf: &[u8]) -> Self {
+            buf.get_u64()
+        }
+    }
+}
diff --git a/src/policies/lru.rs b/src/policies/lru.rs
index 374d3f91..8ec776aa 100644
--- a/src/policies/lru.rs
+++ b/src/policies/lru.rs
@@ -29,8 +29,7 @@
 use std::ptr::NonNull;
 use std::time::SystemTime;
 
-use crate::collections::dlist::{DList, Entry, Iter};
-use crate::intrusive_dlist;
+use crate::collections::dlist::{intrusive_dlist, DList, Entry, Iter};
 
 use super::Index;
 
diff --git a/src/policies/tinylfu.rs b/src/policies/tinylfu.rs
index d690ecba..a8b2d70a 100644
--- a/src/policies/tinylfu.rs
+++ b/src/policies/tinylfu.rs
@@ -33,8 +33,7 @@ use std::time::SystemTime;
 use cmsketch::CMSketchUsize;
 use twox_hash::XxHash64;
 
-use crate::collections::dlist::{DList, Entry, Iter};
-use crate::intrusive_dlist;
+use crate::collections::dlist::{intrusive_dlist, DList, Entry, Iter};
 
 use super::Index;
 
diff --git a/src/store/error.rs b/src/store/error.rs
new file mode 100644
index 00000000..81f26d36
--- /dev/null
+++ b/src/store/error.rs
@@ -0,0 +1,29 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("io error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("nix error: {0}")]
+    Nix(#[from] nix::errno::Errno),
+    #[error("unsupported file system, super block magic: {0}")]
+    UnsupportedFilesystem(i64),
+    #[error("invalid slot: {0}")]
+    InvalidSlot(usize),
+    #[error("other error: {0}")]
+    Other(String),
+}
+
+pub type Result<T> = core::result::Result<T, Error>;
diff --git a/src/store/file.rs b/src/store/file.rs
new file mode 100644
index 00000000..dffcacf7
--- /dev/null
+++ b/src/store/file.rs
@@ -0,0 +1,400 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use bytes::{Buf, BufMut};
+use nix::sys::stat::fstat;
+use nix::sys::uio::{pread, pwrite};
+
+use super::asyncify;
+use super::error::Result;
+
+use std::fs::{remove_file, File, OpenOptions};
+use std::os::fd::{AsRawFd, RawFd};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+#[derive(Clone, Copy, Debug)]
+pub struct Location {
+    pub offset: u32,
+    pub len: u32,
+}
+
+impl Location {
+    pub fn write(&self, mut buf: &mut [u8]) {
+        buf.put_u32_le(self.offset);
+        buf.put_u32_le(self.len);
+    }
+
+    pub fn read(mut buf: &[u8]) -> Self {
+        let offset = buf.get_u32();
+        let len = buf.get_u32();
+        Self { offset, len }
+    }
+
+    pub fn size() -> usize {
+        8
+    }
+}
+
+// TODO(MrCroxx): Use buffer and direct I/O to better manage memory usage.
+pub struct AppendableFile {
+    path: PathBuf,
+
+    fd: RawFd,
+
+    size: AtomicUsize,
+
+    file: File,
+}
+
+impl AppendableFile {
+    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
+        let path = PathBuf::from(path.as_ref());
+
+        let opts = {
+            let mut opts = OpenOptions::new();
+            opts.create(true);
+            opts.write(true);
+            opts.read(true);
+            opts
+        };
+
+        let (file, fd, size) = asyncify({
+            let path = path.clone();
+            move || {
+                let file = opts.open(path)?;
+                let fd = file.as_raw_fd();
+                let stat = fstat(fd)?;
+                let size = stat.st_size as usize;
+                Ok((file, fd, size))
+            }
+        })
+        .await?;
+
+        Ok(Self {
+            path,
+            fd,
+            size: AtomicUsize::new(size),
+            file,
+        })
+    }
+
+    pub async fn append(&self, buf: Vec<u8>) -> Result<Location> {
+        let len = buf.len();
+        let offset = self.size.fetch_add(len, Ordering::Relaxed);
+        let fd = self.fd;
+        asyncify(move || {
+            pwrite(fd, &buf, offset as i64)?;
+            Ok(())
+        })
+        .await?;
+
+        Ok(Location {
+            offset: offset as u32,
+            len: len as u32,
+        })
+    }
+
+    #[allow(clippy::uninit_vec)]
+    pub async fn read(&self, offset: u64, len: usize) -> Result<Vec<u8>> {
+        let fd = self.fd;
+        let buf = asyncify(move || {
+            let mut buf = Vec::with_capacity(len);
+            unsafe { buf.set_len(len) };
+            pread(fd, &mut buf, offset as i64)?;
+            Ok(buf)
+        })
+        .await?;
+        Ok(buf)
+    }
+
+    pub fn len(&self) -> usize {
+        self.size.load(Ordering::Relaxed)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub async fn size(&self) -> Result<usize> {
+        let fd = self.fd;
+        let size = asyncify(move || {
+            let stat = fstat(fd)?;
+            Ok(stat.st_size as usize)
+        })
+        .await?;
+        Ok(size)
+    }
+
+    pub async fn reclaim(self) -> Result<()> {
+        let file = self.file;
+        drop(file);
+
+        asyncify(move || {
+            remove_file(self.path)?;
+            Ok(())
+        })
+        .await
+    }
+}
+
+// TODO(MrCroxx): Use buffer and direct I/O to better manage memory usage.
+pub struct ReadableFile {
+    path: PathBuf,
+
+    fd: RawFd,
+
+    size: usize,
+
+    file: File,
+}
+
+impl ReadableFile {
+    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
+        let path = PathBuf::from(path.as_ref());
+
+        let opts = {
+            let mut opts = OpenOptions::new();
+            opts.read(true);
+            opts
+        };
+
+        let (file, fd, size) = asyncify({
+            let path = path.clone();
+            move || {
+                let file = opts.open(path)?;
+                let fd = file.as_raw_fd();
+                let stat = fstat(fd)?;
+                let size = stat.st_size as usize;
+                Ok((file, fd, size))
+            }
+        })
+        .await?;
+
+        Ok(Self {
+            path,
+            fd,
+            size,
+            file,
+        })
+    }
+
+    #[allow(clippy::uninit_vec)]
+    pub async fn read(&self, offset: u64, len: usize) -> Result<Vec<u8>> {
+        let fd = self.fd;
+        let buf = asyncify(move || {
+            let mut buf = Vec::with_capacity(len);
+            unsafe { buf.set_len(len) };
+            pread(fd, &mut buf, offset as i64)?;
+            Ok(buf)
+        })
+        .await?;
+        Ok(buf)
+    }
+
+    pub fn len(&self) -> usize {
+        self.size
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub async fn size(&self) -> Result<usize> {
+        let fd = self.fd;
+        let size = asyncify(move || {
+            let stat = fstat(fd)?;
+            Ok(stat.st_size as usize)
+        })
+        .await?;
+        Ok(size)
+    }
+
+    pub async fn reclaim(self) -> Result<()> {
+        let file = self.file;
+        drop(file);
+
+        asyncify(move || {
+            remove_file(self.path)?;
+            Ok(())
+        })
+        .await
+    }
+}
+
+pub struct WritableFile {
+    path: PathBuf,
+
+    fd: RawFd,
+
+    size: AtomicUsize,
+
+    file: File,
+}
+
+impl WritableFile {
+    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
+        let path = PathBuf::from(path.as_ref());
+
+        let opts = {
+            let mut opts = OpenOptions::new();
+            opts.create(true);
+            opts.write(true);
+            opts.read(true);
+            opts
+        };
+
+        let (file, fd, size) = asyncify({
+            let path = path.clone();
+            move || {
+                let file = opts.open(path)?;
+                let fd = file.as_raw_fd();
+                let stat = fstat(fd)?;
+                let size = stat.st_size as usize;
+                Ok((file, fd, size))
+            }
+        })
+        .await?;
+
+        Ok(Self {
+            path,
+            fd,
+            size: AtomicUsize::new(size),
+            file,
+        })
+    }
+
+    pub async fn append(&self, buf: Vec<u8>) -> Result<Location> {
+        let len = buf.len();
+        let offset = self.size.fetch_add(len, Ordering::Relaxed);
+        let fd = self.fd;
+        asyncify(move || {
+            pwrite(fd, &buf, offset as i64)?;
+            Ok(())
+        })
+        .await?;
+
+        Ok(Location {
+            offset: offset as u32,
+            len: len as u32,
+        })
+    }
+
+    /// # Safety
+    ///
+    /// Concurrent writes or appends on the same address are not guaranteed.
+    pub async fn write(&self, offset: u64, buf: Vec<u8>) -> Result<Location> {
+        let fd = self.fd;
+        let len = buf.len();
+        asyncify(move || {
+            pwrite(fd, &buf, offset as i64)?;
+            Ok(())
+        })
+        .await?;
+        Ok(Location {
+            offset: offset as u32,
+            len: len as u32,
+        })
+    }
+
+    #[allow(clippy::uninit_vec)]
+    pub async fn read(&self, offset: u64, len: usize) -> Result<Vec<u8>> {
+        let fd = self.fd;
+        let buf = asyncify(move || {
+            let mut buf = Vec::with_capacity(len);
+            unsafe { buf.set_len(len) };
+            pread(fd, &mut buf, offset as i64)?;
+            Ok(buf)
+        })
+        .await?;
+        Ok(buf)
+    }
+
+    pub async fn size(&self) -> Result<usize> {
+        let fd = self.fd;
+        let size = asyncify(move || {
+            let stat = fstat(fd)?;
+            Ok(stat.st_size as usize)
+        })
+        .await?;
+        Ok(size)
+    }
+
+    pub async fn reclaim(self) -> Result<()> {
+        let file = self.file;
+        drop(file);
+
+        asyncify(move || {
+            remove_file(self.path)?;
+            Ok(())
+        })
+        .await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::tests::is_send_sync_static;
+
+    use super::*;
+
+    use futures::future::try_join_all;
+    use itertools::Itertools;
+    use tempfile::tempdir;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn test_rw() {
+        is_send_sync_static::<AppendableFile>();
+        is_send_sync_static::<ReadableFile>();
+
+        let dir = tempdir().unwrap();
+        let path = PathBuf::from(dir.path()).join("testfile");
+
+        let afile = AppendableFile::open(&path).await.unwrap();
+
+        let bufs = (0..4).map(|i| vec![i as u8; 1024]).collect_vec();
+        let futures = bufs
+            .iter()
+            .map(|buf| afile.append(buf.clone()))
+            .collect_vec();
+        let locs = try_join_all(futures).await.unwrap();
+
+        assert_eq!(afile.len(), 4 * 1024);
+        assert_eq!(afile.size().await.unwrap(), 4 * 1024);
+        for (buf, loc) in bufs.iter().zip_eq(locs.iter()) {
+            assert_eq!(
+                buf,
+                &afile
+                    .read(loc.offset as u64, loc.len as usize)
+                    .await
+                    .unwrap()
+            );
+        }
+
+        drop(afile);
+
+        let rfile = ReadableFile::open(&path).await.unwrap();
+        for (buf, loc) in bufs.iter().zip_eq(locs.iter()) {
+            assert_eq!(
+                buf,
+                &rfile
+                    .read(loc.offset as u64, loc.len as usize)
+                    .await
+                    .unwrap()
+            );
+        }
+
+        drop(rfile);
+    }
+}
diff --git a/src/store/mod.rs b/src/store/mod.rs
index eed38f33..d669a58b 100644
--- a/src/store/mod.rs
+++ b/src/store/mod.rs
@@ -12,17 +12,41 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+pub mod error;
+pub mod file;
+pub mod read_only_file_store;
+pub mod utils;
+
 use crate::{Data, Index};
+use async_trait::async_trait;
+
+use error::Result;
 
-pub trait Store: Send + Sync + Clone + 'static {
+#[async_trait]
+pub trait Store: Send + Sync + Sized + 'static {
     type I: Index;
     type D: Data;
+    type C: Send + Sync + Clone + 'static;
+
+    async fn open(pool: usize, config: Self::C) -> Result<Self>;
 
-    fn store(&self, index: Self::I, data: Self::D);
+    async fn store(&self, index: Self::I, data: Self::D) -> Result<()>;
 
-    fn load(&self, index: &Self::I) -> Option<Self::D>;
+    async fn load(&self, index: &Self::I) -> Result<Option<Self::D>>;
 
-    fn delete(&self, index: &Self::I);
+    async fn delete(&self, index: &Self::I) -> Result<()>;
+}
+
+async fn asyncify<F, T>(f: F) -> error::Result<T>
+where
+    F: FnOnce() -> error::Result<T> + Send + 'static,
+    T: Send + 'static,
+{
+    #[cfg_attr(madsim, expect(deprecated))]
+    match tokio::task::spawn_blocking(f).await {
+        Ok(res) => res,
+        Err(_) => Err(error::Error::Other("background task failed".to_string())),
+    }
 }
 
 #[cfg(test)]
@@ -32,39 +56,56 @@ pub mod tests {
 
     use parking_lot::RwLock;
 
+    use super::error::Result;
+
     use super::*;
 
     #[derive(Clone, Debug, Default)]
     pub struct MemoryStore<I: Index, D: Data + Clone> {
+        pool: usize,
         inner: Arc<RwLock<HashMap<I, D>>>,
     }
 
     impl<I: Index, D: Data + Clone> MemoryStore<I, D> {
-        pub fn new() -> Self {
+        pub fn new(pool: usize) -> Self {
             Self {
+                pool,
                 inner: Arc::new(RwLock::new(HashMap::default())),
             }
         }
+
+        pub fn pool(&self) -> usize {
+            self.pool
+        }
     }
 
+    #[async_trait]
     impl<I: Index, D: Data + Clone> Store for MemoryStore<I, D> {
         type I = I;
 
         type D = D;
 
-        fn store(&self, index: Self::I, data: Self::D) {
+        type C = ();
+
+        async fn open(pool: usize, _: Self::C) -> Result<Self> {
+            Ok(Self::new(pool))
+        }
+
+        async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {
             let mut inner = self.inner.write();
             inner.insert(index, data);
+            Ok(())
         }
 
-        fn load(&self, index: &Self::I) -> Option<Self::D> {
+        async fn load(&self, index: &Self::I) -> Result<Option<Self::D>> {
             let inner = self.inner.read();
-            inner.get(index).cloned()
+            Ok(inner.get(index).cloned())
         }
 
-        fn delete(&self, index: &Self::I) {
+        async fn delete(&self, index: &Self::I) -> Result<()> {
             let mut inner = self.inner.write();
             inner.remove(index);
+            Ok(())
         }
     }
 }
diff --git a/src/store/read_only_file_store.rs b/src/store/read_only_file_store.rs
new file mode 100644
index 00000000..bda026bf
--- /dev/null
+++ b/src/store/read_only_file_store.rs
@@ -0,0 +1,377 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::fs::read_dir;
+use std::mem::swap;
+use std::path::{Path, PathBuf};
+
+use std::str::pattern::Pattern;
+use std::sync::Arc;
+use std::{collections::HashMap, marker::PhantomData};
+
+use async_trait::async_trait;
+
+use itertools::Itertools;
+use tokio::sync::{RwLock, RwLockWriteGuard};
+
+use crate::{Data, Index};
+
+use super::error::Result;
+use super::file::{AppendableFile, Location, ReadableFile, WritableFile};
+use super::{asyncify, Store};
+
+pub type FileId = u32;
+pub type SlotId = u32;
+
+const META_FILE_PREFIX: &str = "metafile-";
+const CACHE_FILE_PREFIX: &str = "cachefile-";
+
+#[derive(Clone, Debug)]
+pub struct Config {
+    /// path to store dir
+    pub dir: PathBuf,
+
+    /// max cache file size
+    pub max_file_size: usize,
+}
+
+struct Frozen {
+    fid: FileId,
+    meta_file: WritableFile,
+    cache_file: ReadableFile,
+}
+
+struct Active {
+    fid: FileId,
+    meta_file: WritableFile,
+    cache_file: AppendableFile,
+}
+
+struct ReadOnlyFileStoreFiles {
+    active: Active,
+
+    frozens: HashMap<FileId, Frozen>,
+}
+
+/// `ReadOnlyFileStore` is a file store for read only entries.
+///
+/// The cache data for a cache key MUST NOT be updated.
+#[allow(clippy::type_complexity)]
+pub struct ReadOnlyFileStore<I, D>
+where
+    I: Index,
+    D: Data,
+{
+    pool: usize,
+
+    config: Arc<Config>,
+
+    indices: Arc<RwLock<HashMap<I, (FileId, SlotId, Location)>>>,
+
+    files: Arc<RwLock<ReadOnlyFileStoreFiles>>,
+
+    _marker: PhantomData<D>,
+}
+
+impl<I, D> Clone for ReadOnlyFileStore<I, D>
+where
+    I: Index,
+    D: Data,
+{
+    fn clone(&self) -> Self {
+        Self {
+            pool: self.pool,
+            config: Arc::clone(&self.config),
+            indices: Arc::clone(&self.indices),
+            files: Arc::clone(&self.files),
+            _marker: PhantomData,
+        }
+    }
+}
+
+#[async_trait]
+impl<I, D> Store for ReadOnlyFileStore<I, D>
+where
+    I: Index,
+    D: Data,
+{
+    type I = I;
+
+    type D = D;
+
+    type C = Config;
+
+    async fn open(pool: usize, mut config: Self::C) -> Result<Self> {
+        config.dir = config.dir.join(format!("{:04}", pool));
+
+        let ids = asyncify({
+            let dir = config.dir.clone();
+            move || {
+                let ids: Vec<FileId> = read_dir(dir)?
+                    .map(|entry| entry.unwrap())
+                    .filter(|entry| {
+                        entry
+                            .file_name()
+                            .to_string_lossy()
+                            .is_prefix_of(META_FILE_PREFIX)
+                    })
+                    .map(|entry| {
+                        entry.file_name().to_string_lossy()[META_FILE_PREFIX.len()..].to_string()
+                    })
+                    .map(|s| s.parse().unwrap())
+                    .collect_vec();
+                Ok(ids)
+            }
+        })
+        .await?;
+
+        let mut frozens = HashMap::new();
+        for id in &ids {
+            let frozen = Self::open_frozen_file(&config.dir, *id).await?;
+            frozens.insert(*id, frozen);
+        }
+        let id = ids.into_iter().max().unwrap_or(0) + 1;
+        let active = Self::open_active_file(&config.dir, id).await?;
+
+        let files = ReadOnlyFileStoreFiles { active, frozens };
+        let mut indices = HashMap::new();
+        for (fid, frozen) in &files.frozens {
+            Self::restore_meta(*fid, &frozen.meta_file, &mut indices).await?;
+        }
+
+        Ok(Self {
+            pool,
+            config: Arc::new(config),
+            indices: Arc::new(RwLock::new(indices)),
+            files: Arc::new(RwLock::new(files)),
+            _marker: PhantomData,
+        })
+    }
+
+    #[allow(clippy::uninit_vec)]
+    async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {
+        let buf = data.into();
+
+        // append cache file and meta file
+        let (fid, sid, location) = {
+            let files = self.files.read().await;
+
+            let fid = files.active.fid;
+
+            let location = files.active.cache_file.append(buf).await?;
+
+            let mut buf = Vec::with_capacity(Self::meta_entry_size());
+            unsafe { buf.set_len(Self::meta_entry_size()) };
+            index.write(&mut buf[..]);
+            location.write(&mut buf[I::size()..]);
+            let Location {
+                offset: meta_offset,
+                len: _,
+            } = files.active.meta_file.append(buf).await?;
+            let sid = meta_offset / Self::meta_entry_size() as u32;
+
+            (fid, sid, location)
+        };
+
+        let active_file_size = (location.offset + location.len) as usize;
+
+        {
+            let mut indices = self.indices.write().await;
+            indices.insert(index, (fid, sid, location));
+            drop(indices);
+        }
+
+        if active_file_size >= self.config.max_file_size {
+            let inner = self.files.write().await;
+            // check size again in the critical section to prevent from double rotating
+            if inner.active.cache_file.len() >= self.config.max_file_size {
+                self.rotate_active_file_locked(inner).await?;
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn load(&self, index: &Self::I) -> Result<Option<Self::D>> {
+        // TODO(MrCroxx): add bloom filters ?
+        let (fid, _sid, location) = {
+            let indices = self.indices.read().await;
+
+            let (fid, sid, location) = match indices.get(index) {
+                Some((fid, sid, location)) => (*fid, *sid, *location),
+                None => return Ok(None),
+            };
+
+            (fid, sid, location)
+        };
+
+        let buf = {
+            let files = self.files.read().await;
+
+            if fid == files.active.fid {
+                files
+                    .active
+                    .cache_file
+                    .read(location.offset as u64, location.len as usize)
+                    .await?
+            } else {
+                files
+                    .frozens
+                    .get(&fid)
+                    .expect("frozen file not found")
+                    .cache_file
+                    .read(location.offset as u64, location.len as usize)
+                    .await?
+            }
+        };
+
+        Ok(Some(buf.into()))
+    }
+
+    async fn delete(&self, index: &Self::I) -> Result<()> {
+        let (fid, sid, _location) = {
+            let indices = self.indices.read().await;
+            let (fid, sid, location) = match indices.get(index) {
+                Some((fid, sid, location)) => (*fid, *sid, *location),
+                None => return Ok(()),
+            };
+            (fid, sid, location)
+        };
+
+        {
+            let empty_entry = vec![0; Self::meta_entry_size()];
+            let files = self.files.read().await;
+            if fid == files.active.fid {
+                files
+                    .active
+                    .meta_file
+                    .write(sid as u64 * Self::meta_entry_size() as u64, empty_entry)
+                    .await?;
+            } else {
+                files
+                    .frozens
+                    .get(&fid)
+                    .expect("frozen file not found")
+                    .meta_file
+                    .write(sid as u64 * Self::meta_entry_size() as u64, empty_entry)
+                    .await?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl<I, D> ReadOnlyFileStore<I, D>
+where
+    I: Index,
+    D: Data,
+{
+    async fn rotate_active_file_locked<'a>(
+        &self,
+        mut files: RwLockWriteGuard<'a, ReadOnlyFileStoreFiles>,
+    ) -> Result<()> {
+        // rotate active file
+        let mut active = Self::open_active_file(&self.config.dir, files.active.fid + 1).await?;
+        swap(&mut active, &mut files.active);
+
+        // open frozen file
+        let id = active.fid;
+        let frozen = Self::open_frozen_file(&self.config.dir, id).await?;
+
+        // update frozen map
+        files.frozens.insert(id, frozen);
+
+        Ok(())
+    }
+
+    pub async fn reclaim_frozen_file(&self, id: FileId) -> Result<()> {
+        let (fid, meta_file, cache_file) = {
+            let mut files = self.files.write().await;
+
+            let Frozen {
+                fid,
+                meta_file,
+                cache_file,
+            } = files.frozens.remove(&id).expect("frozen id not exists");
+
+            (fid, meta_file, cache_file)
+        };
+
+        let mut indices_to_delete = HashMap::new();
+        Self::restore_meta(fid, &meta_file, &mut indices_to_delete).await?;
+
+        {
+            let mut indices = self.indices.write().await;
+            for index in indices_to_delete.keys() {
+                indices.remove(index);
+            }
+        }
+
+        meta_file.reclaim().await?;
+        cache_file.reclaim().await?;
+
+        Ok(())
+    }
+
+    async fn open_active_file(dir: impl AsRef<Path>, id: FileId) -> Result<Active> {
+        let meta_file = WritableFile::open(Self::meta_file_path(&dir, id)).await?;
+        let cache_file = AppendableFile::open(Self::cache_file_path(&dir, id)).await?;
+        Ok(Active {
+            fid: id,
+            meta_file,
+            cache_file,
+        })
+    }
+
+    async fn open_frozen_file(dir: impl AsRef<Path>, id: FileId) -> Result<Frozen> {
+        let meta_file = WritableFile::open(Self::meta_file_path(&dir, id)).await?;
+        let cache_file = ReadableFile::open(Self::cache_file_path(&dir, id)).await?;
+
+        Ok(Frozen {
+            fid: id,
+            meta_file,
+            cache_file,
+        })
+    }
+
+    async fn restore_meta(
+        fid: FileId,
+        meta: &WritableFile,
+        indices: &mut HashMap<I, (FileId, SlotId, Location)>,
+    ) -> Result<()> {
+        let size = meta.size().await?;
+        let slots = size / Self::meta_entry_size();
+        let buf = meta.read(0, size).await?;
+        for sid in 0..slots {
+            let slice = &buf[sid * Self::meta_entry_size()..(sid + 1) * Self::meta_entry_size()];
+            let index = I::read(slice);
+            let location = Location::read(&slice[I::size()..]);
+            indices.insert(index, (fid, sid as SlotId, location));
+        }
+        Ok(())
+    }
+
+    fn cache_file_path(dir: impl AsRef<Path>, id: FileId) -> PathBuf {
+        PathBuf::from(dir.as_ref()).join(format!("{}{:08}", META_FILE_PREFIX, id))
+    }
+
+    fn meta_file_path(dir: impl AsRef<Path>, id: FileId) -> PathBuf {
+        PathBuf::from(dir.as_ref()).join(format!("{}{:08}", CACHE_FILE_PREFIX, id))
+    }
+
+    fn meta_entry_size() -> usize {
+        I::size() + Location::size()
+    }
+}
diff --git a/src/store/utils.rs b/src/store/utils.rs
new file mode 100644
index 00000000..8db5e622
--- /dev/null
+++ b/src/store/utils.rs
@@ -0,0 +1,111 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{Debug, Display};
+use std::ops::{Add, BitAnd, Not, Sub};
+
+pub trait UnsignedTrait = Add<Output = Self>
+    + Sub<Output = Self>
+    + BitAnd<Output = Self>
+    + Not<Output = Self>
+    + Sized
+    + From<u8>
+    + Eq
+    + Debug
+    + Display
+    + Clone
+    + Copy;
+
+pub trait Unsigned: UnsignedTrait {}
+
+impl<U: UnsignedTrait> Unsigned for U {}
+
+#[inline(always)]
+pub fn is_pow2<U: Unsigned>(v: U) -> bool {
+    v & (v - U::from(1)) == U::from(0)
+}
+
+#[inline(always)]
+pub fn assert_pow2<U: Unsigned>(v: U) {
+    assert_eq!(v & (v - U::from(1)), U::from(0), "v: {}", v);
+}
+
+#[inline(always)]
+pub fn debug_assert_pow2<U: Unsigned>(v: U) {
+    debug_assert_eq!(v & (v - U::from(1)), U::from(0), "v: {}", v);
+}
+
+#[inline(always)]
+pub fn is_aligned<U: Unsigned>(align: U, v: U) -> bool {
+    debug_assert_pow2(align);
+    v & (align - U::from(1)) == U::from(0)
+}
+
+#[inline(always)]
+pub fn assert_aligned<U: Unsigned>(align: U, v: U) {
+    debug_assert_pow2(align);
+    assert!(is_aligned(align, v), "align: {}, v: {}", align, v);
+}
+
+#[inline(always)]
+pub fn debug_assert_aligned<U: Unsigned>(align: U, v: U) {
+    debug_assert_pow2(align);
+    debug_assert!(is_aligned(align, v), "align: {}, v: {}", align, v);
+}
+
+#[inline(always)]
+pub fn align_up<U: Unsigned>(align: U, v: U) -> U {
+    debug_assert_pow2(align);
+    (v + align - U::from(1)) & !(align - U::from(1))
+}
+
+#[inline(always)]
+pub fn align_down<U: Unsigned>(align: U, v: U) -> U {
+    debug_assert_pow2(align);
+    v & !(align - U::from(1))
+}
+
+// macro_rules! bpf_buffer_trace {
+//     ($buf:expr, $span:expr) => {
+//         #[cfg(feature = "bpf")]
+//         {
+//             if $buf.len() >= 16 && let Some(id) = $span.id() {
+//                 use bytes::BufMut;
+
+//                 const BPF_BUFFER_TRACE_MAGIC: u64 = 0xdeadbeefdeadbeef;
+
+//                 $span.record("sid", id.into_u64());
+
+//                 (&mut $buf[0..8]).put_u64_le(BPF_BUFFER_TRACE_MAGIC);
+//                 (&mut $buf[8..16]).put_u64_le(id.into_u64());
+//             }
+//         }
+//     };
+// }
+
+// pub(crate) use bpf_buffer_trace;

From 7a2467ca340cb99d395f37a8e94437ad387e8191 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 25 May 2023 11:23:03 +0800
Subject: [PATCH 009/261] fix: add simple test for read only file store and fix
 bugs (#10)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 src/lib.rs                        |   4 +-
 src/store/file.rs                 |   4 +-
 src/store/read_only_file_store.rs | 160 ++++++++++++++++++++++++++++--
 3 files changed, 155 insertions(+), 13 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index e84b421d..c29c28c6 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -15,6 +15,8 @@
 #![feature(trait_alias)]
 #![feature(pattern)]
 
+use std::fmt::Debug;
+
 pub use policies::Policy;
 
 pub mod collections;
@@ -23,7 +25,7 @@ pub mod policies;
 pub mod store;
 
 pub trait Index:
-    PartialOrd + Ord + PartialEq + Eq + Clone + std::hash::Hash + Send + Sync + 'static
+    PartialOrd + Ord + PartialEq + Eq + Clone + std::hash::Hash + Send + Sync + 'static + Debug
 {
     fn size() -> usize;
 
diff --git a/src/store/file.rs b/src/store/file.rs
index dffcacf7..bc9bdc15 100644
--- a/src/store/file.rs
+++ b/src/store/file.rs
@@ -37,8 +37,8 @@ impl Location {
     }
 
     pub fn read(mut buf: &[u8]) -> Self {
-        let offset = buf.get_u32();
-        let len = buf.get_u32();
+        let offset = buf.get_u32_le();
+        let len = buf.get_u32_le();
         Self { offset, len }
     }
 
diff --git a/src/store/read_only_file_store.rs b/src/store/read_only_file_store.rs
index bda026bf..9bc82bbb 100644
--- a/src/store/read_only_file_store.rs
+++ b/src/store/read_only_file_store.rs
@@ -12,11 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::fs::read_dir;
+use std::fs::{create_dir_all, read_dir};
 use std::mem::swap;
 use std::path::{Path, PathBuf};
 
 use std::str::pattern::Pattern;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::{collections::HashMap, marker::PhantomData};
 
@@ -44,6 +45,21 @@ pub struct Config {
 
     /// max cache file size
     pub max_file_size: usize,
+
+    /// store capacity
+    pub capacity: usize,
+
+    /// ratio of garbage to trigger reclaim
+    pub trigger_reclaim_garbage_ratio: f64,
+
+    /// ratio of size to trigger reclaim
+    pub trigger_reclaim_capacity_ratio: f64,
+
+    /// ratio of size to trigger randomly drop
+    pub trigger_random_drop_ratio: f64,
+
+    /// ratio of randomly dropped entries
+    pub random_drop_ratio: f64,
 }
 
 struct Frozen {
@@ -79,8 +95,11 @@ where
 
     indices: Arc<RwLock<HashMap<I, (FileId, SlotId, Location)>>>,
 
+    /// write lock is used when rotating active file or reclaiming frozen files
     files: Arc<RwLock<ReadOnlyFileStoreFiles>>,
 
+    size: Arc<AtomicUsize>,
+
     _marker: PhantomData<D>,
 }
 
@@ -95,6 +114,7 @@ where
             config: Arc::clone(&self.config),
             indices: Arc::clone(&self.indices),
             files: Arc::clone(&self.files),
+            size: Arc::clone(&self.size),
             _marker: PhantomData,
         }
     }
@@ -118,6 +138,7 @@ where
         let ids = asyncify({
             let dir = config.dir.clone();
             move || {
+                create_dir_all(&dir)?;
                 let ids: Vec<FileId> = read_dir(dir)?
                     .map(|entry| entry.unwrap())
                     .filter(|entry| {
@@ -137,10 +158,14 @@ where
         .await?;
 
         let mut frozens = HashMap::new();
+        let mut size = 0;
         for id in &ids {
             let frozen = Self::open_frozen_file(&config.dir, *id).await?;
+            // when restore, `len` is filled with `fstat(2)`
+            size += frozen.cache_file.len();
             frozens.insert(*id, frozen);
         }
+        // create new active file on every `open`
         let id = ids.into_iter().max().unwrap_or(0) + 1;
         let active = Self::open_active_file(&config.dir, id).await?;
 
@@ -155,6 +180,7 @@ where
             config: Arc::new(config),
             indices: Arc::new(RwLock::new(indices)),
             files: Arc::new(RwLock::new(files)),
+            size: Arc::new(AtomicUsize::new(size)),
             _marker: PhantomData,
         })
     }
@@ -162,6 +188,7 @@ where
     #[allow(clippy::uninit_vec)]
     async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {
         let buf = data.into();
+        let len = buf.len();
 
         // append cache file and meta file
         let (fid, sid, location) = {
@@ -192,14 +219,18 @@ where
             drop(indices);
         }
 
+        self.size.fetch_add(len, Ordering::Relaxed);
+
         if active_file_size >= self.config.max_file_size {
-            let inner = self.files.write().await;
+            let files = self.files.write().await;
             // check size again in the critical section to prevent from double rotating
-            if inner.active.cache_file.len() >= self.config.max_file_size {
-                self.rotate_active_file_locked(inner).await?;
+            if files.active.cache_file.len() >= self.config.max_file_size {
+                self.rotate_active_file_locked(files).await?;
             }
         }
 
+        self.maybe_trigger_reclaim().await?;
+
         Ok(())
     }
 
@@ -236,11 +267,13 @@ where
             }
         };
 
+        self.maybe_trigger_reclaim().await?;
+
         Ok(Some(buf.into()))
     }
 
     async fn delete(&self, index: &Self::I) -> Result<()> {
-        let (fid, sid, _location) = {
+        let (fid, sid, location) = {
             let indices = self.indices.read().await;
             let (fid, sid, location) = match indices.get(index) {
                 Some((fid, sid, location)) => (*fid, *sid, *location),
@@ -269,6 +302,11 @@ where
             }
         }
 
+        self.size
+            .fetch_sub(location.len as usize, Ordering::Relaxed);
+
+        self.maybe_trigger_reclaim().await?;
+
         Ok(())
     }
 }
@@ -296,7 +334,7 @@ where
         Ok(())
     }
 
-    pub async fn reclaim_frozen_file(&self, id: FileId) -> Result<()> {
+    async fn reclaim_frozen_file(&self, id: FileId) -> Result<()> {
         let (fid, meta_file, cache_file) = {
             let mut files = self.files.write().await;
 
@@ -312,12 +350,17 @@ where
         let mut indices_to_delete = HashMap::new();
         Self::restore_meta(fid, &meta_file, &mut indices_to_delete).await?;
 
-        {
+        let size = {
+            let mut size = 0;
             let mut indices = self.indices.write().await;
-            for index in indices_to_delete.keys() {
-                indices.remove(index);
+            for (index, (_fid, _sid, Location { offset: _, len })) in indices_to_delete {
+                indices.remove(&index);
+                size += len;
             }
-        }
+            size as usize
+        };
+
+        self.size.fetch_sub(size, Ordering::Relaxed);
 
         meta_file.reclaim().await?;
         cache_file.reclaim().await?;
@@ -325,6 +368,41 @@ where
         Ok(())
     }
 
+    async fn maybe_trigger_reclaim(&self) -> Result<()> {
+        // trigger by size ratio
+        if self.size.load(Ordering::Relaxed) as f64
+            >= self.config.capacity as f64 * self.config.trigger_reclaim_capacity_ratio
+        {
+            self.reclaim().await?;
+        }
+
+        // TODO(MrCroxx): trigger reclaim based on garbage ratio
+
+        Ok(())
+    }
+
+    /// Reclaim garbage to make room.
+    ///
+    /// Policy:
+    ///
+    /// [WIP] For now, simply reclaim the oldest frozen file.
+    ///
+    /// TODO(MrCroxx): better reclaim policy
+    async fn reclaim(&self) -> Result<()> {
+        let id = {
+            let files = self.files.read().await;
+            let id = files.frozens.keys().min();
+            match id {
+                Some(id) => *id,
+                None => return Ok(()),
+            }
+        };
+
+        self.reclaim_frozen_file(id).await?;
+
+        Ok(())
+    }
+
     async fn open_active_file(dir: impl AsRef<Path>, id: FileId) -> Result<Active> {
         let meta_file = WritableFile::open(Self::meta_file_path(&dir, id)).await?;
         let cache_file = AppendableFile::open(Self::cache_file_path(&dir, id)).await?;
@@ -375,3 +453,65 @@ where
         I::size() + Location::size()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+
+    fn data(i: u8, len: usize) -> Vec<u8> {
+        vec![i; len]
+    }
+
+    #[tokio::test]
+    async fn test_read_only_file_store_simple() {
+        let dir = tempdir().unwrap();
+
+        let config = Config {
+            dir: dir.path().to_owned(),
+            max_file_size: 4 * 1024,
+            capacity: 16 * 1024,
+            trigger_reclaim_garbage_ratio: 0.0, // disabled
+            trigger_reclaim_capacity_ratio: 0.75,
+            trigger_random_drop_ratio: 0.0, // disabled
+            random_drop_ratio: 0.0,         // disabled
+        };
+
+        let store: ReadOnlyFileStore<u64, Vec<u8>> =
+            ReadOnlyFileStore::open(0, config).await.unwrap();
+
+        store.store(1, data(1, 1024)).await.unwrap();
+        assert_eq!(store.load(&1).await.unwrap(), Some(data(1, 1024)));
+
+        store.store(2, data(2, 1024)).await.unwrap();
+        assert_eq!(store.load(&2).await.unwrap(), Some(data(2, 1024)));
+        store.store(3, data(3, 1024)).await.unwrap();
+        assert_eq!(store.load(&3).await.unwrap(), Some(data(3, 1024)));
+        store.store(4, data(4, 1024)).await.unwrap();
+        assert_eq!(store.load(&4).await.unwrap(), Some(data(4, 1024)));
+
+        // assert rotate
+        assert_eq!(store.files.read().await.frozens.len(), 1);
+
+        assert_eq!(store.load(&1).await.unwrap(), Some(data(1, 1024)));
+        assert_eq!(store.load(&2).await.unwrap(), Some(data(2, 1024)));
+        assert_eq!(store.load(&3).await.unwrap(), Some(data(3, 1024)));
+        assert_eq!(store.load(&4).await.unwrap(), Some(data(4, 1024)));
+
+        store.store(5, data(5, 4 * 1024)).await.unwrap();
+        assert_eq!(store.size.load(Ordering::Relaxed), 8 * 1024);
+        assert_eq!(store.files.read().await.frozens.len(), 2);
+
+        // assert reclaim
+        store.store(6, data(6, 4 * 1024)).await.unwrap();
+        assert_eq!(store.files.read().await.frozens.len(), 2);
+        assert_eq!(store.size.load(Ordering::Relaxed), 8 * 1024);
+
+        assert_eq!(store.load(&1).await.unwrap(), None);
+        assert_eq!(store.load(&2).await.unwrap(), None);
+        assert_eq!(store.load(&3).await.unwrap(), None);
+        assert_eq!(store.load(&4).await.unwrap(), None);
+
+        drop(dir);
+    }
+}

From d6a7a285ab9bf8d04453c52a05077619eeb0faa2 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 26 May 2023 12:30:29 +0800
Subject: [PATCH 010/261] feat: add foyer bench, reorg workspace (#11)

* feat: add foyer bench, reorg workspace

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* sort cargo file

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* make fmt and clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix bug

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .cargo/config.toml                            |   2 +
 Cargo.lock                                    | 247 ++++++++++++
 Cargo.toml                                    |  36 +-
 foyer-bench/Cargo.toml                        |  30 ++
 foyer-bench/src/analyze.rs                    | 260 ++++++++++++
 foyer-bench/src/main.rs                       | 371 ++++++++++++++++++
 foyer-bench/src/rate.rs                       |  45 +++
 foyer-bench/src/utils.rs                      | 153 ++++++++
 foyer/Cargo.toml                              |  39 ++
 {src => foyer/src}/collections/dlist.rs       |   0
 {src => foyer/src}/collections/mod.rs         |   0
 {src => foyer/src}/container.rs               |  73 ++--
 {src => foyer/src}/lib.rs                     |  69 +++-
 {src => foyer/src}/policies/lru.rs            |   9 +-
 {src => foyer/src}/policies/mod.rs            |   0
 {src => foyer/src}/policies/tinylfu.rs        |   9 +-
 {src => foyer/src}/store/error.rs             |   0
 {src => foyer/src}/store/file.rs              |   0
 {src => foyer/src}/store/mod.rs               |   0
 .../src}/store/read_only_file_store.rs        |   6 +-
 {src => foyer/src}/store/utils.rs             |   0
 21 files changed, 1244 insertions(+), 105 deletions(-)
 create mode 100644 .cargo/config.toml
 create mode 100644 foyer-bench/Cargo.toml
 create mode 100644 foyer-bench/src/analyze.rs
 create mode 100644 foyer-bench/src/main.rs
 create mode 100644 foyer-bench/src/rate.rs
 create mode 100644 foyer-bench/src/utils.rs
 create mode 100644 foyer/Cargo.toml
 rename {src => foyer/src}/collections/dlist.rs (100%)
 rename {src => foyer/src}/collections/mod.rs (100%)
 rename {src => foyer/src}/container.rs (82%)
 rename {src => foyer/src}/lib.rs (51%)
 rename {src => foyer/src}/policies/lru.rs (98%)
 rename {src => foyer/src}/policies/mod.rs (100%)
 rename {src => foyer/src}/policies/tinylfu.rs (99%)
 rename {src => foyer/src}/store/error.rs (100%)
 rename {src => foyer/src}/store/file.rs (100%)
 rename {src => foyer/src}/store/mod.rs (100%)
 rename {src => foyer/src}/store/read_only_file_store.rs (100%)
 rename {src => foyer/src}/store/utils.rs (100%)

diff --git a/.cargo/config.toml b/.cargo/config.toml
new file mode 100644
index 00000000..73ac2e51
--- /dev/null
+++ b/.cargo/config.toml
@@ -0,0 +1,2 @@
+[target.'cfg(all())']
+rustflags = ["--cfg", "tokio_unstable"]
diff --git a/Cargo.lock b/Cargo.lock
index df8f2062..cdae3f61 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,61 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "anstream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is-terminal",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
+dependencies = [
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.68"
@@ -19,18 +74,36 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
 [[package]]
 name = "bytes"
 version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
 
+[[package]]
+name = "bytesize"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38fcc2979eff34a4b84e1cf9a1e3da42a7d44b3b690a40cdcb23e3d556cfb2e5"
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -43,6 +116,48 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "clap"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+ "once_cell",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "bitflags",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+
 [[package]]
 name = "cmsketch"
 version = "0.1.4"
@@ -52,6 +167,21 @@ dependencies = [
  "paste",
 ]
 
+[[package]]
+name = "colorchoice"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
+
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crossbeam"
 version = "0.8.2"
@@ -155,21 +285,35 @@ dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "flate2"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "foyer"
 version = "0.1.0"
 dependencies = [
  "async-trait",
  "bytes",
+ "bytesize",
+ "clap",
  "cmsketch",
  "crossbeam",
  "futures",
+ "hdrhistogram",
  "itertools",
  "libc",
  "memoffset 0.8.0",
  "nix",
  "parking_lot",
  "paste",
+ "rand",
  "rand_mt",
  "tempfile",
  "thiserror",
@@ -177,6 +321,25 @@ dependencies = [
  "twox-hash",
 ]
 
+[[package]]
+name = "foyer-bench"
+version = "0.1.0"
+dependencies = [
+ "bytesize",
+ "clap",
+ "foyer",
+ "futures",
+ "hdrhistogram",
+ "itertools",
+ "libc",
+ "nix",
+ "parking_lot",
+ "rand",
+ "rand_mt",
+ "tempfile",
+ "tokio",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.28"
@@ -277,6 +440,26 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "hdrhistogram"
+version = "7.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8"
+dependencies = [
+ "base64",
+ "byteorder",
+ "crossbeam-channel",
+ "flate2",
+ "nom",
+ "num-traits",
+]
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "hermit-abi"
 version = "0.2.6"
@@ -312,6 +495,18 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "is-terminal"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
+dependencies = [
+ "hermit-abi 0.3.1",
+ "io-lifetimes",
+ "rustix",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -376,6 +571,21 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.6"
@@ -402,6 +612,25 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.15.0"
@@ -412,6 +641,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -584,6 +819,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
 [[package]]
 name = "syn"
 version = "2.0.15"
@@ -672,6 +913,12 @@ version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
diff --git a/Cargo.toml b/Cargo.toml
index 7ddea64c..6b03ac89 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,35 +1,3 @@
-[package]
-name = "foyer"
-version = "0.1.0"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
-license = "Apache-2.0"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[workspace]
 
-[dependencies]
-async-trait = "0.1"
-bytes = "1"
-cmsketch = "0.1"
-crossbeam = "0.8"
-futures = "0.3"
-itertools = "0.10.5"
-libc = "0.2"
-memoffset = "0.8"
-nix = { version = "0.26", features = ["fs", "mman"] }
-parking_lot = "0.12"
-paste = "1.0"
-thiserror = "1"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
-twox-hash = "1"
-
-[dev-dependencies]
-rand_mt = "4.2.1"
-tempfile = "3"
+members = ["foyer", "foyer-bench"]
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
new file mode 100644
index 00000000..241b1933
--- /dev/null
+++ b/foyer-bench/Cargo.toml
@@ -0,0 +1,30 @@
+[package]
+name = "foyer-bench"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+foyer = { path = "../foyer" }
+futures = "0.3"
+hdrhistogram = "7"
+itertools = "0.10.5"
+libc = "0.2"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+rand = "0.8.5"
+rand_mt = "4.2.1"
+tempfile = "3"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
diff --git a/foyer-bench/src/analyze.rs b/foyer-bench/src/analyze.rs
new file mode 100644
index 00000000..4b4be02e
--- /dev/null
+++ b/foyer-bench/src/analyze.rs
@@ -0,0 +1,260 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use bytesize::ByteSize;
+use hdrhistogram::Histogram;
+use parking_lot::RwLock;
+use tokio::sync::oneshot;
+
+use crate::utils::{iostat, IoStat};
+
+const SECTOR_SIZE: usize = 512;
+
+// latencies are measured by 'us'
+#[derive(Clone, Copy, Debug)]
+pub struct Analysis {
+    disk_read_iops: f64,
+    disk_read_throughput: f64,
+    disk_write_iops: f64,
+    disk_write_throughput: f64,
+
+    insert_iops: f64,
+    insert_throughput: f64,
+    insert_lat_p50: u64,
+    insert_lat_p90: u64,
+    insert_lat_p99: u64,
+
+    get_iops: f64,
+    get_miss: f64,
+    get_miss_lat_p50: u64,
+    get_miss_lat_p90: u64,
+    get_miss_lat_p99: u64,
+    get_hit_lat_p50: u64,
+    get_hit_lat_p90: u64,
+    get_hit_lat_p99: u64,
+}
+
+#[derive(Default, Clone, Copy, Debug)]
+pub struct MetricsDump {
+    pub insert_ios: usize,
+    pub insert_bytes: usize,
+    pub insert_lat_p50: u64,
+    pub insert_lat_p90: u64,
+    pub insert_lat_p99: u64,
+
+    pub get_ios: usize,
+    pub get_miss_ios: usize,
+    pub get_hit_lat_p50: u64,
+    pub get_hit_lat_p90: u64,
+    pub get_hit_lat_p99: u64,
+    pub get_miss_lat_p50: u64,
+    pub get_miss_lat_p90: u64,
+    pub get_miss_lat_p99: u64,
+}
+
+#[derive(Clone, Debug)]
+pub struct Metrics {
+    pub insert_ios: Arc<AtomicUsize>,
+    pub insert_bytes: Arc<AtomicUsize>,
+    pub insert_lats: Arc<RwLock<Histogram<u64>>>,
+
+    pub get_ios: Arc<AtomicUsize>,
+    pub get_miss_ios: Arc<AtomicUsize>,
+    pub get_hit_lats: Arc<RwLock<Histogram<u64>>>,
+    pub get_miss_lats: Arc<RwLock<Histogram<u64>>>,
+}
+
+impl Default for Metrics {
+    fn default() -> Self {
+        Self {
+            insert_ios: Arc::new(AtomicUsize::new(0)),
+            insert_bytes: Arc::new(AtomicUsize::new(0)),
+            insert_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+
+            get_ios: Arc::new(AtomicUsize::new(0)),
+            get_miss_ios: Arc::new(AtomicUsize::new(0)),
+            get_hit_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+            get_miss_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+        }
+    }
+}
+
+impl Metrics {
+    pub fn dump(&self) -> MetricsDump {
+        let insert_lats = self.insert_lats.read();
+        let get_hit_lats = self.get_hit_lats.read();
+        let get_miss_lats = self.get_miss_lats.read();
+        MetricsDump {
+            insert_ios: self.insert_ios.load(Ordering::Relaxed),
+            insert_bytes: self.insert_bytes.load(Ordering::Relaxed),
+            insert_lat_p50: insert_lats.value_at_quantile(0.5),
+            insert_lat_p90: insert_lats.value_at_quantile(0.9),
+            insert_lat_p99: insert_lats.value_at_quantile(0.99),
+
+            get_ios: self.get_ios.load(Ordering::Relaxed),
+            get_miss_ios: self.get_miss_ios.load(Ordering::Relaxed),
+            get_hit_lat_p50: get_hit_lats.value_at_quantile(0.5),
+            get_hit_lat_p90: get_hit_lats.value_at_quantile(0.9),
+            get_hit_lat_p99: get_hit_lats.value_at_quantile(0.99),
+            get_miss_lat_p50: get_miss_lats.value_at_quantile(0.5),
+            get_miss_lat_p90: get_miss_lats.value_at_quantile(0.9),
+            get_miss_lat_p99: get_miss_lats.value_at_quantile(0.99),
+        }
+    }
+}
+
+impl std::fmt::Display for Analysis {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let disk_read_throughput = ByteSize::b(self.disk_read_throughput as u64);
+        let disk_write_throughput = ByteSize::b(self.disk_write_throughput as u64);
+        let disk_total_throughput = disk_read_throughput + disk_write_throughput;
+
+        // disk statics
+        writeln!(
+            f,
+            "disk total iops: {:.1}",
+            self.disk_write_iops + self.disk_read_iops
+        )?;
+        writeln!(
+            f,
+            "disk total throughput: {}/s",
+            disk_total_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "disk read iops: {:.1}", self.disk_read_iops)?;
+        writeln!(
+            f,
+            "disk read throughput: {}/s",
+            disk_read_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "disk write iops: {:.1}", self.disk_write_iops)?;
+        writeln!(
+            f,
+            "disk write throughput: {}/s",
+            disk_write_throughput.to_string_as(true)
+        )?;
+
+        // insert statics
+        let insert_throughput = ByteSize::b(self.insert_throughput as u64);
+        writeln!(f, "insert iops: {:.1}/s", self.insert_iops)?;
+        writeln!(
+            f,
+            "insert throughput: {}/s",
+            insert_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
+        writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
+        writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
+
+        // get statics
+        writeln!(f, "get iops: {:.1}/s", self.get_iops)?;
+        writeln!(f, "get miss: {:.2}% ", self.get_miss * 100f64)?;
+        writeln!(f, "get hit lat p50: {}us", self.get_hit_lat_p50)?;
+        writeln!(f, "get hit lat p90: {}us", self.get_hit_lat_p90)?;
+        writeln!(f, "get hit lat p99: {}us", self.get_hit_lat_p99)?;
+        writeln!(f, "get miss lat p50: {}us", self.get_miss_lat_p50)?;
+        writeln!(f, "get miss lat p90: {}us", self.get_miss_lat_p90)?;
+        writeln!(f, "get miss lat p99: {}us", self.get_miss_lat_p99)?;
+
+        Ok(())
+    }
+}
+
+pub fn analyze(
+    duration: Duration,
+    iostat_start: &IoStat,
+    iostat_end: &IoStat,
+    metrics_dump_start: &MetricsDump,
+    metrics_dump_end: &MetricsDump,
+) -> Analysis {
+    let secs = duration.as_secs_f64();
+    let disk_read_iops = (iostat_end.read_ios - iostat_start.read_ios) as f64 / secs;
+    let disk_read_throughput =
+        (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+    let disk_write_iops = (iostat_end.write_ios - iostat_start.write_ios) as f64 / secs;
+    let disk_write_throughput =
+        (iostat_end.write_sectors - iostat_start.write_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+
+    let insert_iops = (metrics_dump_end.insert_ios - metrics_dump_start.insert_ios) as f64 / secs;
+    let insert_throughput =
+        (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
+
+    let get_iops = (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64 / secs;
+    let get_miss = (metrics_dump_end.get_miss_ios - metrics_dump_start.get_miss_ios) as f64
+        / (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64;
+
+    Analysis {
+        disk_read_iops,
+        disk_read_throughput,
+        disk_write_iops,
+        disk_write_throughput,
+
+        insert_iops,
+        insert_throughput,
+        insert_lat_p50: metrics_dump_end.insert_lat_p50,
+        insert_lat_p90: metrics_dump_end.insert_lat_p90,
+        insert_lat_p99: metrics_dump_end.insert_lat_p99,
+
+        get_iops,
+        get_miss,
+        get_hit_lat_p50: metrics_dump_end.get_hit_lat_p50,
+        get_hit_lat_p90: metrics_dump_end.get_hit_lat_p90,
+        get_hit_lat_p99: metrics_dump_end.get_hit_lat_p99,
+        get_miss_lat_p50: metrics_dump_end.get_miss_lat_p50,
+        get_miss_lat_p90: metrics_dump_end.get_miss_lat_p90,
+        get_miss_lat_p99: metrics_dump_end.get_miss_lat_p99,
+    }
+}
+
+pub async fn monitor(
+    iostat_path: impl AsRef<Path>,
+    interval: Duration,
+    metrics: Metrics,
+    mut stop: oneshot::Receiver<()>,
+) {
+    let mut stat = iostat(&iostat_path);
+    let mut metrics_dump = metrics.dump();
+    loop {
+        let start = Instant::now();
+        match stop.try_recv() {
+            Err(oneshot::error::TryRecvError::Empty) => {}
+            _ => return,
+        }
+
+        tokio::time::sleep(interval).await;
+        let new_stat = iostat(&iostat_path);
+        let new_metrics_dump = metrics.dump();
+        let analysis = analyze(
+            // interval may have ~ +7% error
+            start.elapsed(),
+            &stat,
+            &new_stat,
+            &metrics_dump,
+            &new_metrics_dump,
+        );
+        println!("{}", analysis);
+        stat = new_stat;
+        metrics_dump = new_metrics_dump;
+    }
+}
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
new file mode 100644
index 00000000..dd6aa918
--- /dev/null
+++ b/foyer-bench/src/main.rs
@@ -0,0 +1,371 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(let_chains)]
+
+mod analyze;
+mod rate;
+mod utils;
+
+use std::fs::create_dir_all;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use analyze::{analyze, monitor, Metrics};
+use clap::Parser;
+
+use foyer::container::{Config, Container};
+
+use foyer::store::read_only_file_store::{Config as ReadOnlyFileStoreConfig, ReadOnlyFileStore};
+
+use foyer::policies::tinylfu::{Config as TinyLfuConfig, Handle, TinyLfu};
+use futures::future::join_all;
+use itertools::Itertools;
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+
+use rate::RateLimiter;
+use tokio::sync::oneshot;
+use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
+
+#[derive(Parser, Debug, Clone)]
+pub struct Args {
+    /// dir for cache data
+    #[arg(short, long)]
+    dir: String,
+
+    /// (MiB)
+    #[arg(long)]
+    capacity: usize,
+
+    /// (s)
+    #[arg(short, long)]
+    time: u64,
+
+    #[arg(long)]
+    concurrency: usize,
+
+    /// must be power of 2
+    #[arg(long)]
+    pools: usize,
+
+    /// (s)
+    #[arg(long, default_value_t = 2)]
+    report_interval: u64,
+
+    /// Some filesystem (e.g. btrfs) can span across multiple block devices and it's hard to decide
+    /// which device to moitor. Use this argument to specify which block device to monitor.
+    #[arg(long, default_value = "")]
+    iostat_dev: String,
+
+    #[arg(long, default_value_t = 0.0)]
+    w_rate: f64,
+
+    #[arg(long, default_value_t = 0.0)]
+    r_rate: f64,
+
+    #[arg(long, default_value_t = 64 * 1024)]
+    entry_size: usize,
+
+    #[arg(long, default_value_t = 10000)]
+    look_up_range: u64,
+
+    /// read only file store: max cache file size (MiB)
+    #[arg(long, default_value_t = 64)]
+    rofs_max_file_size: usize,
+
+    /// read only file store: ratio of garbage to trigger reclaim (0 ~ 1)
+    #[arg(long, default_value_t = 0.0)]
+    rofs_trigger_reclaim_garbage_ratio: f64,
+
+    /// read only file store: ratio of size to trigger reclaim (0 ~ 1)
+    #[arg(long, default_value_t = 0.8)]
+    rofs_trigger_reclaim_capacity_ratio: f64,
+
+    /// read only file store: ratio of size to trigger randomly drop (0 ~ 1)
+    #[arg(long, default_value_t = 0.0)]
+    rofs_trigger_random_drop_ratio: f64,
+
+    /// read only file store: ratio of randomly dropped entries (0 ~ 1)
+    #[arg(long, default_value_t = 0.0)]
+    rofs_random_drop_ratio: f64,
+}
+
+impl Args {
+    fn verify(&self) {
+        assert!(self.pools.is_power_of_two());
+    }
+}
+
+type TContainer =
+    Container<u64, TinyLfu<u64>, Handle<u64>, Vec<u8>, ReadOnlyFileStore<u64, Vec<u8>>>;
+
+fn is_send_sync_static<T: Send + Sync + 'static>() {}
+
+#[tokio::main]
+async fn main() {
+    is_send_sync_static::<TContainer>();
+
+    let args = Args::parse();
+    args.verify();
+
+    create_dir_all(&args.dir).unwrap();
+
+    let iostat_path = match detect_fs_type(&args.dir) {
+        FsType::Tmpfs => panic!("tmpfs is not supported with benches"),
+        FsType::Btrfs => {
+            if args.iostat_dev.is_empty() {
+                panic!("cannot decide which block device to monitor for btrfs, please specify device name with \'--iostat-dev\'")
+            } else {
+                dev_stat_path(&args.iostat_dev)
+            }
+        }
+        _ => file_stat_path(&args.dir),
+    };
+
+    let metrics = Metrics::default();
+
+    let iostat_start = iostat(&iostat_path);
+    let metrics_dump_start = metrics.dump();
+    let time = Instant::now();
+
+    let store_config = ReadOnlyFileStoreConfig {
+        dir: PathBuf::from(&args.dir),
+        capacity: args.capacity * 1024 * 1024,
+        max_file_size: args.rofs_max_file_size * 1024 * 1024,
+        trigger_reclaim_garbage_ratio: args.rofs_trigger_reclaim_garbage_ratio,
+        trigger_reclaim_capacity_ratio: args.rofs_trigger_reclaim_capacity_ratio,
+        trigger_random_drop_ratio: args.rofs_trigger_random_drop_ratio,
+        random_drop_ratio: args.rofs_random_drop_ratio,
+    };
+
+    let policy_config = TinyLfuConfig {
+        window_to_cache_size_ratio: 1,
+        tiny_lru_capacity_ratio: 0.01,
+    };
+
+    let config: Config<u64, TinyLfu<_>, _, ReadOnlyFileStore<_, Vec<u8>>> = Config {
+        capacity: args.capacity * 1024 * 1024,
+        pool_count_bits: (args.pools as f64).log2() as usize,
+        policy_config,
+        store_config,
+    };
+
+    let container = Container::open(config).await.unwrap();
+    let container = Arc::new(container);
+
+    let (iostat_stop_tx, iostat_stop_rx) = oneshot::channel();
+    let (bench_stop_txs, bench_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
+        (0..args.concurrency).map(|_| oneshot::channel()).unzip();
+
+    let handle_monitor = tokio::spawn({
+        let iostat_path = iostat_path.clone();
+        let metrics = metrics.clone();
+
+        monitor(
+            iostat_path,
+            Duration::from_secs(args.report_interval),
+            metrics,
+            iostat_stop_rx,
+        )
+    });
+    let handle_signal = tokio::spawn(async move {
+        tokio::signal::ctrl_c().await.unwrap();
+        for bench_stop_tx in bench_stop_txs {
+            bench_stop_tx.send(()).unwrap();
+        }
+        iostat_stop_tx.send(()).unwrap();
+    });
+    let handles = (bench_stop_rxs)
+        .into_iter()
+        .map(|stop| {
+            tokio::spawn(bench(
+                args.clone(),
+                container.clone(),
+                metrics.clone(),
+                stop,
+            ))
+        })
+        .collect_vec();
+
+    for handle in handles {
+        handle.await.unwrap();
+    }
+    handle_monitor.abort();
+    handle_signal.abort();
+
+    let iostat_end = iostat(&iostat_path);
+    let metrics_dump_end = metrics.dump();
+    let analysis = analyze(
+        time.elapsed(),
+        &iostat_start,
+        &iostat_end,
+        &metrics_dump_start,
+        &metrics_dump_end,
+    );
+    println!("\nTotal:\n{}", analysis);
+}
+
+async fn bench(
+    args: Args,
+    container: Arc<TContainer>,
+    metrics: Metrics,
+    stop: oneshot::Receiver<()>,
+) {
+    let w_rate = if args.w_rate == 0.0 {
+        None
+    } else {
+        Some(args.w_rate)
+    };
+    let r_rate = if args.r_rate == 0.0 {
+        None
+    } else {
+        Some(args.r_rate)
+    };
+
+    let index = Arc::new(AtomicU64::new(0));
+
+    let (w_stop_tx, w_stop_rx) = oneshot::channel();
+    let (r_stop_tx, r_stop_rx) = oneshot::channel();
+
+    let handle_w = tokio::spawn(write(
+        args.entry_size,
+        w_rate,
+        index.clone(),
+        container.clone(),
+        args.time,
+        metrics.clone(),
+        w_stop_rx,
+    ));
+    let handle_r = tokio::spawn(read(
+        args.entry_size,
+        r_rate,
+        index.clone(),
+        container.clone(),
+        args.time,
+        metrics.clone(),
+        r_stop_rx,
+        args.look_up_range,
+    ));
+    tokio::spawn(async move {
+        if let Ok(()) = stop.await {
+            let _ = w_stop_tx.send(());
+            let _ = r_stop_tx.send(());
+        }
+    });
+
+    join_all([handle_r, handle_w]).await;
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn write(
+    entry_size: usize,
+    rate: Option<f64>,
+    index: Arc<AtomicU64>,
+    container: Arc<TContainer>,
+    time: u64,
+    metrics: Metrics,
+    mut stop: oneshot::Receiver<()>,
+) {
+    let start = Instant::now();
+
+    let mut limiter = rate.map(RateLimiter::new);
+
+    loop {
+        match stop.try_recv() {
+            Err(oneshot::error::TryRecvError::Empty) => {}
+            _ => return,
+        }
+        if start.elapsed().as_secs() >= time {
+            return;
+        }
+
+        let idx = index.fetch_add(1, Ordering::Relaxed);
+        // TODO(MrCroxx): Use random content?
+        let data = vec![b'x'; entry_size];
+        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
+            tokio::time::sleep(wait).await;
+        }
+
+        let time = Instant::now();
+        container.insert(idx, data).await.unwrap();
+        metrics
+            .insert_lats
+            .write()
+            .record(time.elapsed().as_micros() as u64)
+            .expect("record out of range");
+        metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
+        metrics
+            .insert_bytes
+            .fetch_add(entry_size, Ordering::Relaxed);
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn read(
+    entry_size: usize,
+    rate: Option<f64>,
+    index: Arc<AtomicU64>,
+    container: Arc<TContainer>,
+    time: u64,
+    metrics: Metrics,
+    mut stop: oneshot::Receiver<()>,
+    look_up_range: u64,
+) {
+    let start = Instant::now();
+
+    let mut limiter = rate.map(RateLimiter::new);
+
+    let mut rng = StdRng::seed_from_u64(0);
+
+    loop {
+        match stop.try_recv() {
+            Err(oneshot::error::TryRecvError::Empty) => {}
+            _ => return,
+        }
+        if start.elapsed().as_secs() >= time {
+            return;
+        }
+
+        let idx_max = index.load(Ordering::Relaxed);
+        let idx = rng.gen_range(std::cmp::max(idx_max, look_up_range) - look_up_range..=idx_max);
+
+        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
+            tokio::time::sleep(wait).await;
+        }
+
+        let time = Instant::now();
+        let hit = container.get(&idx).await.unwrap().is_some();
+        let lat = time.elapsed().as_micros() as u64;
+        if hit {
+            metrics
+                .get_hit_lats
+                .write()
+                .record(lat)
+                .expect("record out of range");
+        } else {
+            metrics
+                .get_miss_lats
+                .write()
+                .record(lat)
+                .expect("record out of range");
+            metrics.get_miss_ios.fetch_add(1, Ordering::Relaxed);
+        }
+        metrics.get_ios.fetch_add(1, Ordering::Relaxed);
+
+        tokio::task::consume_budget().await;
+    }
+}
diff --git a/foyer-bench/src/rate.rs b/foyer-bench/src/rate.rs
new file mode 100644
index 00000000..a2d0853a
--- /dev/null
+++ b/foyer-bench/src/rate.rs
@@ -0,0 +1,45 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::{Duration, Instant};
+
+pub struct RateLimiter {
+    capacity: f64,
+    quota: f64,
+
+    last: Instant,
+}
+
+impl RateLimiter {
+    pub fn new(capacity: f64) -> Self {
+        Self {
+            capacity,
+            quota: 0.0,
+            last: Instant::now(),
+        }
+    }
+
+    pub fn consume(&mut self, weight: f64) -> Option<Duration> {
+        let now = Instant::now();
+        let refill = now.duration_since(self.last).as_secs_f64() * self.capacity;
+        self.last = now;
+        self.quota = f64::min(self.quota + refill, self.capacity);
+        self.quota -= weight;
+        if self.quota >= 0.0 {
+            return None;
+        }
+        let wait = Duration::from_secs_f64((-self.quota) / self.capacity);
+        Some(wait)
+    }
+}
diff --git a/foyer-bench/src/utils.rs b/foyer-bench/src/utils.rs
new file mode 100644
index 00000000..0b2a7ffe
--- /dev/null
+++ b/foyer-bench/src/utils.rs
@@ -0,0 +1,153 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::{Path, PathBuf};
+
+use itertools::Itertools;
+use nix::fcntl::readlink;
+use nix::sys::stat::stat;
+use nix::sys::statfs::{statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC};
+
+#[derive(PartialEq, Clone, Copy, Debug)]
+pub enum FsType {
+    Xfs,
+    Ext4,
+    Btrfs,
+    Tmpfs,
+    Others,
+}
+
+pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
+    #[cfg(target_os = "linux")]
+    {
+        let fs_stat = statfs(path.as_ref()).unwrap();
+        match fs_stat.filesystem_type() {
+            XFS_SUPER_MAGIC => FsType::Xfs,
+            EXT4_SUPER_MAGIC => FsType::Ext4,
+            BTRFS_SUPER_MAGIC => FsType::Btrfs,
+            TMPFS_MAGIC => FsType::Tmpfs,
+            _ => FsType::Others,
+        }
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    FsType::Others
+}
+
+/// Given a normal file path, returns the containing block device static file path (of the
+/// partition).
+pub fn file_stat_path(path: impl AsRef<Path>) -> PathBuf {
+    let st_dev = stat(path.as_ref()).unwrap().st_dev;
+
+    let major = unsafe { libc::major(st_dev) };
+    let minor = unsafe { libc::minor(st_dev) };
+
+    let dev = PathBuf::from("/dev/block").join(format!("{}:{}", major, minor));
+
+    let linkname = readlink(&dev).unwrap();
+    let devname = Path::new(linkname.as_os_str()).file_name().unwrap();
+    dev_stat_path(devname.to_str().unwrap())
+}
+
+pub fn dev_stat_path(devname: &str) -> PathBuf {
+    let classpath = Path::new("/sys/class/block").join(devname);
+    let devclass = readlink(&classpath).unwrap();
+
+    let devpath = Path::new(&devclass);
+    Path::new("/sys")
+        .join(devpath.strip_prefix("../..").unwrap())
+        .join("stat")
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct IoStat {
+    /// read I/Os       requests      number of read I/Os processed
+    pub read_ios: usize,
+    /// read merges     requests      number of read I/Os merged with in-queue I/O
+    pub read_merges: usize,
+    /// read sectors    sectors       number of sectors read
+    pub read_sectors: usize,
+    /// read ticks      milliseconds  total wait time for read requests
+    pub read_ticks: usize,
+    /// write I/Os      requests      number of write I/Os processed
+    pub write_ios: usize,
+    /// write merges    requests      number of write I/Os merged with in-queue I/O
+    pub write_merges: usize,
+    /// write sectors   sectors       number of sectors written
+    pub write_sectors: usize,
+    /// write ticks     milliseconds  total wait time for write requests
+    pub write_ticks: usize,
+    /// in_flight       requests      number of I/Os currently in flight
+    pub in_flight: usize,
+    /// io_ticks        milliseconds  total time this block device has been active
+    pub io_ticks: usize,
+    /// time_in_queue   milliseconds  total wait time for all requests
+    pub time_in_queue: usize,
+    /// discard I/Os    requests      number of discard I/Os processed
+    pub discard_ios: usize,
+    /// discard merges  requests      number of discard I/Os merged with in-queue I/O
+    pub discard_merges: usize,
+    /// discard sectors sectors       number of sectors discarded
+    pub discard_sectors: usize,
+    /// discard ticks   milliseconds  total wait time for discard requests
+    pub discard_ticks: usize,
+    /// flush I/Os      requests      number of flush I/Os processed
+    pub flush_ios: usize,
+    /// flush ticks     milliseconds  total wait time for flush requests
+    pub flush_ticks: usize,
+}
+
+/// Given the device static file path and get the iostat.
+pub fn iostat(path: impl AsRef<Path>) -> IoStat {
+    let content = std::fs::read_to_string(path.as_ref()).unwrap();
+    let nums = content.split_ascii_whitespace().collect_vec();
+
+    let read_ios = nums[0].parse().unwrap();
+    let read_merges = nums[1].parse().unwrap();
+    let read_sectors = nums[2].parse().unwrap();
+    let read_ticks = nums[3].parse().unwrap();
+    let write_ios = nums[4].parse().unwrap();
+    let write_merges = nums[5].parse().unwrap();
+    let write_sectors = nums[6].parse().unwrap();
+    let write_ticks = nums[7].parse().unwrap();
+    let in_flight = nums[8].parse().unwrap();
+    let io_ticks = nums[9].parse().unwrap();
+    let time_in_queue = nums[10].parse().unwrap();
+    let discard_ios = nums[11].parse().unwrap();
+    let discard_merges = nums[12].parse().unwrap();
+    let discard_sectors = nums[13].parse().unwrap();
+    let discard_ticks = nums[14].parse().unwrap();
+    let flush_ios = nums[15].parse().unwrap();
+    let flush_ticks = nums[16].parse().unwrap();
+
+    IoStat {
+        read_ios,
+        read_merges,
+        read_sectors,
+        read_ticks,
+        write_ios,
+        write_merges,
+        write_sectors,
+        write_ticks,
+        in_flight,
+        io_ticks,
+        time_in_queue,
+        discard_ios,
+        discard_merges,
+        discard_sectors,
+        discard_ticks,
+        flush_ios,
+        flush_ticks,
+    }
+}
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
new file mode 100644
index 00000000..69bdec42
--- /dev/null
+++ b/foyer/Cargo.toml
@@ -0,0 +1,39 @@
+[package]
+name = "foyer"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1"
+bytes = "1"
+cmsketch = "0.1"
+crossbeam = "0.8"
+futures = "0.3"
+itertools = "0.10.5"
+libc = "0.2"
+memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+paste = "1.0"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+twox-hash = "1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+hdrhistogram = "7"
+rand = "0.8.5"
+rand_mt = "4.2.1"
+tempfile = "3"
diff --git a/src/collections/dlist.rs b/foyer/src/collections/dlist.rs
similarity index 100%
rename from src/collections/dlist.rs
rename to foyer/src/collections/dlist.rs
diff --git a/src/collections/mod.rs b/foyer/src/collections/mod.rs
similarity index 100%
rename from src/collections/mod.rs
rename to foyer/src/collections/mod.rs
diff --git a/src/container.rs b/foyer/src/container.rs
similarity index 82%
rename from src/container.rs
rename to foyer/src/container.rs
index caa9aefb..a56dcc8f 100644
--- a/src/container.rs
+++ b/foyer/src/container.rs
@@ -14,7 +14,7 @@
 
 use std::collections::BTreeMap;
 use std::hash::Hasher;
-use std::marker::PhantomData;
+
 use std::ptr::NonNull;
 
 use futures::future::try_join_all;
@@ -24,58 +24,50 @@ use twox_hash::XxHash64;
 
 use crate::policies::{Handle, Policy};
 use crate::store::Store;
-use crate::{Data, Index};
+use crate::{Data, Index, WrappedNonNull};
 
 // TODO(MrCroxx): wrap own result type
 use crate::store::error::Result;
 
-pub struct Config<I, P, H, T, W, S>
+pub struct Config<I, P, H, S>
 where
     I: Index,
     P: Policy<I = I, H = H>,
     H: Handle<I = I>,
-    W: Fn(&T) -> usize + Send + Sync,
-    S: Store<I = I, D = T>,
+    S: Store<I = I>,
 {
-    capacity: usize,
-
-    pool_count_bits: usize,
-
-    policy_config: P::C,
+    pub capacity: usize,
 
-    store_config: S::C,
+    pub pool_count_bits: usize,
 
-    weighter: W,
+    pub policy_config: P::C,
 
-    _marker: PhantomData<(I, H, T)>,
+    pub store_config: S::C,
 }
 
 #[allow(clippy::type_complexity)]
-pub struct Container<I, P, H, D, W, S>
+pub struct Container<I, P, H, D, S>
 where
     I: Index,
     P: Policy<I = I, H = H>,
     H: Handle<I = I>,
     D: Data,
-    W: Fn(&D) -> usize + Send + Sync,
     S: Store<I = I, D = D>,
 {
     pool_count_bits: usize,
     pools: Vec<Mutex<Pool<I, P, H, D, S>>>,
-
-    weighter: W,
 }
 
-impl<I, P, H, D, W, S> Container<I, P, H, D, W, S>
+impl<I, P, H, D, S> Container<I, P, H, D, S>
 where
     I: Index,
     P: Policy<I = I, H = H>,
     H: Handle<I = I>,
     D: Data,
-    W: Fn(&D) -> usize + Send + Sync,
+
     S: Store<I = I, D = D>,
 {
-    pub async fn open(config: Config<I, P, H, D, W, S>) -> Result<Self> {
+    pub async fn open(config: Config<I, P, H, S>) -> Result<Self> {
         let pool_count = 1 << config.pool_count_bits;
         let capacity = config.capacity >> config.pool_count_bits;
 
@@ -101,8 +93,6 @@ where
         Ok(Self {
             pool_count_bits: config.pool_count_bits,
             pools,
-
-            weighter: config.weighter,
         })
     }
 
@@ -114,7 +104,7 @@ where
             return Ok(false);
         }
 
-        let weight = (self.weighter)(&data);
+        let weight = data.weight();
 
         pool.make_room(weight).await?;
 
@@ -196,23 +186,23 @@ where
             if self.size + weight <= self.capacity || self.size == 0 {
                 break;
             }
-            let PoolHandle { weight, handle } = self.handles.remove(index).unwrap();
-            self.size -= weight;
-            handles.push(handle);
+            let pool_handle = self.handles.remove(index).unwrap();
+            self.size -= pool_handle.weight;
+            handles.push(pool_handle.handle);
             self.store.delete(index).await?;
         }
         for handle in handles {
-            assert!(self.policy.remove(handle));
-            unsafe { drop(Box::from_raw(handle.as_ptr())) };
+            assert!(self.policy.remove(handle.0));
+            unsafe { drop(Box::from_raw(handle.0.as_ptr())) };
         }
         Ok(())
     }
 
     async fn insert(&mut self, index: I, weight: usize, data: D) -> Result<()> {
         let handle = Box::new(H::new(index.clone()));
-        let handle = unsafe { NonNull::new_unchecked(Box::leak(handle)) };
+        let handle = unsafe { WrappedNonNull(NonNull::new_unchecked(Box::leak(handle))) };
 
-        assert!(self.policy.insert(handle));
+        assert!(self.policy.insert(handle.0));
         self.handles
             .insert(index.clone(), PoolHandle { weight, handle });
         self.size += weight;
@@ -224,7 +214,7 @@ where
 
     async fn remove(&mut self, index: &I) -> Result<()> {
         let PoolHandle { weight, handle } = self.handles.remove(index).unwrap();
-        assert!(self.policy.remove(handle));
+        assert!(self.policy.remove(handle.0));
         self.size -= weight;
 
         self.store.delete(index).await?;
@@ -234,8 +224,8 @@ where
 
     async fn get(&mut self, index: &I) -> Result<Option<D>> {
         match self.handles.get(index) {
-            Some(PoolHandle { weight: _, handle }) => {
-                self.policy.access(*handle);
+            Some(pool_handle) => {
+                self.policy.access(pool_handle.handle.0);
                 self.store.load(index).await
             }
             None => Ok(None),
@@ -249,7 +239,8 @@ where
     H: Handle<I = I>,
 {
     weight: usize,
-    handle: NonNull<H>,
+    // Use `WrappedNonNull` for ptrs that needs to cross .await points.
+    handle: WrappedNonNull<H>,
 }
 
 #[cfg(test)]
@@ -257,13 +248,16 @@ mod tests {
     use super::*;
 
     use crate::policies::lru::{Config as LruConfig, Handle as LruHandle, Lru};
+    use crate::policies::tinylfu::Handle as TinyLfuHandle;
     use crate::store::tests::MemoryStore;
+    use crate::tests::is_send_sync_static;
 
     #[tokio::test]
     async fn test_container_simple() {
+        is_send_sync_static::<PoolHandle<u64, LruHandle<u64>>>();
+        is_send_sync_static::<PoolHandle<u64, TinyLfuHandle<u64>>>();
+
         let policy_config = LruConfig {
-            update_on_write: true,
-            update_on_read: true,
             lru_insertion_point_fraction: 0.0,
         };
 
@@ -271,12 +265,11 @@ mod tests {
             capacity: 100,
             pool_count_bits: 0,
             policy_config,
-            weighter: |data: &Vec<u8>| data.len(),
+
             store_config: (),
-            _marker: PhantomData,
         };
 
-        let container: Container<u64, Lru<_>, LruHandle<_>, Vec<u8>, _, MemoryStore<_, _>> =
+        let container: Container<u64, Lru<_>, LruHandle<_>, Vec<u8>, MemoryStore<_, _>> =
             Container::open(config).await.unwrap();
 
         assert!(container.insert(1, vec![b'x'; 40]).await.unwrap());
@@ -287,7 +280,9 @@ mod tests {
         assert!(!container.insert(2, vec![b'x'; 60]).await.unwrap());
         assert_eq!(container.get(&2).await.unwrap(), Some(vec![b'x'; 60]));
 
+        // After insert 3, {1, 2} will be evicted.
         assert!(container.insert(3, vec![b'x'; 50]).await.unwrap());
+        assert_eq!(container.size().await, 50);
         assert_eq!(container.get(&3).await.unwrap(), Some(vec![b'x'; 50]));
         assert_eq!(container.get(&1).await.unwrap(), None);
         assert_eq!(container.get(&2).await.unwrap(), None);
diff --git a/src/lib.rs b/foyer/src/lib.rs
similarity index 51%
rename from src/lib.rs
rename to foyer/src/lib.rs
index c29c28c6..47fd517f 100644
--- a/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -15,15 +15,27 @@
 #![feature(trait_alias)]
 #![feature(pattern)]
 
-use std::fmt::Debug;
+use std::{fmt::Debug, ptr::NonNull};
 
 pub use policies::Policy;
 
+use paste::paste;
+
 pub mod collections;
 pub mod container;
 pub mod policies;
 pub mod store;
 
+pub trait Weight {
+    fn weight(&self) -> usize;
+}
+
+impl Weight for Vec<u8> {
+    fn weight(&self) -> usize {
+        self.len()
+    }
+}
+
 pub trait Index:
     PartialOrd + Ord + PartialEq + Eq + Clone + std::hash::Hash + Send + Sync + 'static + Debug
 {
@@ -33,28 +45,51 @@ pub trait Index:
 
     fn read(buf: &[u8]) -> Self;
 }
-pub trait Data = Send + Sync + 'static + Into<Vec<u8>> + From<Vec<u8>>;
+pub trait Data = Send + Sync + 'static + Into<Vec<u8>> + From<Vec<u8>> + Weight;
 
-#[cfg(test)]
+macro_rules! for_all_primitives {
+    ($macro:ident) => {
+        $macro! {
+            u8, u16, u32, u64,
+            i8, i16, i32, i64,
+        }
+    };
+}
 
-pub mod tests {
-    use bytes::{Buf, BufMut};
+macro_rules! impl_index {
+    ($( $type:ty, )*) => {
+        use bytes::{Buf, BufMut};
 
-    use super::*;
+        paste! {
+            $(
+                impl Index for $type {
+                    fn size() -> usize {
+                        std::mem::size_of::<$type>()
+                    }
 
-    pub fn is_send_sync_static<T: Send + Sync + 'static>() {}
+                    fn write(&self, mut buf: &mut [u8]) {
+                        buf.[< put_ $type>](*self)
+                    }
 
-    impl Index for u64 {
-        fn size() -> usize {
-            8
+                    fn read(mut buf: &[u8]) -> Self {
+                        buf.[< get_ $type>]()
+                    }
+                }
+            )*
         }
+    };
+}
 
-        fn write(&self, mut buf: &mut [u8]) {
-            buf.put_u64(*self)
-        }
+for_all_primitives! { impl_index }
 
-        fn read(mut buf: &[u8]) -> Self {
-            buf.get_u64()
-        }
-    }
+pub struct WrappedNonNull<T>(pub NonNull<T>);
+
+unsafe impl<T> Send for WrappedNonNull<T> {}
+unsafe impl<T> Sync for WrappedNonNull<T> {}
+
+#[cfg(test)]
+
+pub mod tests {
+
+    pub fn is_send_sync_static<T: Send + Sync + 'static>() {}
 }
diff --git a/src/policies/lru.rs b/foyer/src/policies/lru.rs
similarity index 98%
rename from src/policies/lru.rs
rename to foyer/src/policies/lru.rs
index 8ec776aa..50a53353 100644
--- a/src/policies/lru.rs
+++ b/foyer/src/policies/lru.rs
@@ -35,10 +35,6 @@ use super::Index;
 
 #[derive(Clone, Debug)]
 pub struct Config {
-    pub update_on_write: bool,
-
-    pub update_on_read: bool,
-
     /// Insertion point of the new entry, between 0 and 1.
     pub lru_insertion_point_fraction: f64,
 }
@@ -262,6 +258,9 @@ unsafe impl<I: Index> Sync for Lru<I> {}
 unsafe impl<I: Index> Send for Handle<I> {}
 unsafe impl<I: Index> Sync for Handle<I> {}
 
+unsafe impl<'a, I: Index> Send for EvictionIter<'a, I> {}
+unsafe impl<'a, I: Index> Sync for EvictionIter<'a, I> {}
+
 impl super::Config for Config {}
 
 impl<I: Index> super::Handle for Handle<I> {
@@ -316,8 +315,6 @@ mod tests {
     #[test]
     fn test_lru_simple() {
         let config = Config {
-            update_on_write: true,
-            update_on_read: true,
             lru_insertion_point_fraction: 0.0,
         };
         let mut lru: Lru<u64> = Lru::new(config);
diff --git a/src/policies/mod.rs b/foyer/src/policies/mod.rs
similarity index 100%
rename from src/policies/mod.rs
rename to foyer/src/policies/mod.rs
diff --git a/src/policies/tinylfu.rs b/foyer/src/policies/tinylfu.rs
similarity index 99%
rename from src/policies/tinylfu.rs
rename to foyer/src/policies/tinylfu.rs
index a8b2d70a..a758a997 100644
--- a/src/policies/tinylfu.rs
+++ b/foyer/src/policies/tinylfu.rs
@@ -44,10 +44,6 @@ const DECAY_FACTOR: f64 = 0.5;
 
 #[derive(Clone, Debug)]
 pub struct Config {
-    pub update_on_write: bool,
-
-    pub update_on_read: bool,
-
     /// The multiplier for window len given the cache size.
     pub window_to_cache_size_ratio: usize,
 
@@ -400,6 +396,9 @@ unsafe impl<I: Index> Sync for TinyLfu<I> {}
 unsafe impl<I: Index> Send for Handle<I> {}
 unsafe impl<I: Index> Sync for Handle<I> {}
 
+unsafe impl<'a, I: Index> Send for EvictionIter<'a, I> {}
+unsafe impl<'a, I: Index> Sync for EvictionIter<'a, I> {}
+
 impl super::Config for Config {}
 
 impl<I: Index> super::Handle for Handle<I> {
@@ -454,8 +453,6 @@ mod tests {
     #[test]
     fn test_lru_simple() {
         let config = Config {
-            update_on_write: true,
-            update_on_read: true,
             window_to_cache_size_ratio: 10,
             tiny_lru_capacity_ratio: 0.01,
         };
diff --git a/src/store/error.rs b/foyer/src/store/error.rs
similarity index 100%
rename from src/store/error.rs
rename to foyer/src/store/error.rs
diff --git a/src/store/file.rs b/foyer/src/store/file.rs
similarity index 100%
rename from src/store/file.rs
rename to foyer/src/store/file.rs
diff --git a/src/store/mod.rs b/foyer/src/store/mod.rs
similarity index 100%
rename from src/store/mod.rs
rename to foyer/src/store/mod.rs
diff --git a/src/store/read_only_file_store.rs b/foyer/src/store/read_only_file_store.rs
similarity index 100%
rename from src/store/read_only_file_store.rs
rename to foyer/src/store/read_only_file_store.rs
index 9bc82bbb..8730fbdf 100644
--- a/src/store/read_only_file_store.rs
+++ b/foyer/src/store/read_only_file_store.rs
@@ -43,12 +43,12 @@ pub struct Config {
     /// path to store dir
     pub dir: PathBuf,
 
-    /// max cache file size
-    pub max_file_size: usize,
-
     /// store capacity
     pub capacity: usize,
 
+    /// max cache file size
+    pub max_file_size: usize,
+
     /// ratio of garbage to trigger reclaim
     pub trigger_reclaim_garbage_ratio: f64,
 
diff --git a/src/store/utils.rs b/foyer/src/store/utils.rs
similarity index 100%
rename from src/store/utils.rs
rename to foyer/src/store/utils.rs

From 3e9cc5f4b320137e80e4b7caba204ae4f781a942 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 26 May 2023 14:53:22 +0800
Subject: [PATCH 011/261] fix: fix bug with bench (#12)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-bench/src/main.rs   | 16 ++++++++++------
 foyer/src/container.rs    | 17 +++++++++++++++++
 foyer/src/policies/mod.rs |  2 +-
 foyer/src/store/mod.rs    |  2 +-
 4 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index dd6aa918..3d2a300a 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -48,18 +48,18 @@ pub struct Args {
     dir: String,
 
     /// (MiB)
-    #[arg(long)]
+    #[arg(long, default_value_t = 1024)]
     capacity: usize,
 
     /// (s)
-    #[arg(short, long)]
+    #[arg(short, long, default_value_t = 60)]
     time: u64,
 
-    #[arg(long)]
+    #[arg(long, default_value_t = 16)]
     concurrency: usize,
 
     /// must be power of 2
-    #[arg(long)]
+    #[arg(long, default_value_t = 16)]
     pools: usize,
 
     /// (s)
@@ -84,7 +84,7 @@ pub struct Args {
     look_up_range: u64,
 
     /// read only file store: max cache file size (MiB)
-    #[arg(long, default_value_t = 64)]
+    #[arg(long, default_value_t = 16)]
     rofs_max_file_size: usize,
 
     /// read only file store: ratio of garbage to trigger reclaim (0 ~ 1)
@@ -122,6 +122,8 @@ async fn main() {
     let args = Args::parse();
     args.verify();
 
+    println!("{:#?}", args);
+
     create_dir_all(&args.dir).unwrap();
 
     let iostat_path = match detect_fs_type(&args.dir) {
@@ -144,7 +146,7 @@ async fn main() {
 
     let store_config = ReadOnlyFileStoreConfig {
         dir: PathBuf::from(&args.dir),
-        capacity: args.capacity * 1024 * 1024,
+        capacity: args.capacity * 1024 * 1024 / args.pools,
         max_file_size: args.rofs_max_file_size * 1024 * 1024,
         trigger_reclaim_garbage_ratio: args.rofs_trigger_reclaim_garbage_ratio,
         trigger_reclaim_capacity_ratio: args.rofs_trigger_reclaim_capacity_ratio,
@@ -164,6 +166,8 @@ async fn main() {
         store_config,
     };
 
+    println!("{:#?}", config);
+
     let container = Container::open(config).await.unwrap();
     let container = Arc::new(container);
 
diff --git a/foyer/src/container.rs b/foyer/src/container.rs
index a56dcc8f..59daf695 100644
--- a/foyer/src/container.rs
+++ b/foyer/src/container.rs
@@ -45,6 +45,23 @@ where
     pub store_config: S::C,
 }
 
+impl<I, P, H, S> std::fmt::Debug for Config<I, P, H, S>
+where
+    I: Index,
+    P: Policy<I = I, H = H>,
+    H: Handle<I = I>,
+    S: Store<I = I>,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Config")
+            .field("capacity", &self.capacity)
+            .field("pool_count_bits", &self.pool_count_bits)
+            .field("policy_config", &self.policy_config)
+            .field("store_config", &self.store_config)
+            .finish()
+    }
+}
+
 #[allow(clippy::type_complexity)]
 pub struct Container<I, P, H, D, S>
 where
diff --git a/foyer/src/policies/mod.rs b/foyer/src/policies/mod.rs
index 06d224f1..475640c5 100644
--- a/foyer/src/policies/mod.rs
+++ b/foyer/src/policies/mod.rs
@@ -40,7 +40,7 @@ pub trait Policy: Send + Sync + 'static {
     fn eviction_iter(&self) -> Self::E<'_>;
 }
 
-pub trait Config: Send + Sync + Clone + 'static {}
+pub trait Config: Send + Sync + std::fmt::Debug + Clone + 'static {}
 
 pub trait Handle: Send + Sync + 'static {
     type I: Index;
diff --git a/foyer/src/store/mod.rs b/foyer/src/store/mod.rs
index d669a58b..cbebbaa4 100644
--- a/foyer/src/store/mod.rs
+++ b/foyer/src/store/mod.rs
@@ -26,7 +26,7 @@ use error::Result;
 pub trait Store: Send + Sync + Sized + 'static {
     type I: Index;
     type D: Data;
-    type C: Send + Sync + Clone + 'static;
+    type C: Send + Sync + Clone + std::fmt::Debug + 'static;
 
     async fn open(pool: usize, config: Self::C) -> Result<Self>;
 

From 4d3a8e57d067cf6110c559f0d6614a1861d8c174 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 26 May 2023 16:02:32 +0800
Subject: [PATCH 012/261] ci: add asan test (#13)

* ci: add asan test

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix Cargo.toml and CI

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* regen ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* rename CI step, test asan fail

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* regen CI

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/asan.yml          |  0
 .github/template/template.yml      | 30 ++++++++++++++++++++++++++++++
 .github/workflows/main.yml         | 29 +++++++++++++++++++++++++++++
 .github/workflows/pull-request.yml | 29 +++++++++++++++++++++++++++++
 Cargo.toml                         |  3 +++
 foyer/src/container.rs             | 17 +++++++++++++++++
 6 files changed, 108 insertions(+)
 create mode 100644 .github/template/asan.yml

diff --git a/.github/template/asan.yml b/.github/template/asan.yml
new file mode 100644
index 00000000..e69de29b
diff --git a/.github/template/template.yml b/.github/template/template.yml
index eba6475d..dbf99535 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -70,3 +70,33 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  asan:
+    name: run with address saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          # components: rustfmt, clippy, llvm-tools-preview
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-bench With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+        run: |
+          cargo build --all --target x86_64-unknown-linux-gnu &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data --capacity 256
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index cf0e635f..50abe598 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -77,6 +77,35 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  asan:
+    name: run with address saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-bench With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data --capacity 256
 
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 2463a7ef..b1dca860 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -76,6 +76,35 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  asan:
+    name: run with address saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-bench With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data --capacity 256
 concurrency:
   group: environment-${{ github.ref }}
   cancel-in-progress: true
diff --git a/Cargo.toml b/Cargo.toml
index 6b03ac89..628cc248 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,3 +1,6 @@
 [workspace]
 
 members = ["foyer", "foyer-bench"]
+
+[patch.crates-io]
+# cmsketch = { path = "../cmsketch-rs" }
diff --git a/foyer/src/container.rs b/foyer/src/container.rs
index 59daf695..eff0be4b 100644
--- a/foyer/src/container.rs
+++ b/foyer/src/container.rs
@@ -250,6 +250,23 @@ where
     }
 }
 
+impl<I, P, H, D, S> Drop for Pool<I, P, H, D, S>
+where
+    I: Index,
+    P: Policy<I = I, H = H>,
+    H: Handle<I = I>,
+    D: Data,
+    S: Store<I = I, D = D>,
+{
+    fn drop(&mut self) {
+        let mut handles = BTreeMap::new();
+        std::mem::swap(&mut handles, &mut self.handles);
+        for PoolHandle { weight: _, handle } in handles.into_values() {
+            unsafe { drop(Box::from_raw(handle.0.as_ptr())) }
+        }
+    }
+}
+
 struct PoolHandle<I, H>
 where
     I: Index,

From 5c09e607a013dc684f65941d5ad147a2f51fcdf1 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 26 May 2023 16:35:57 +0800
Subject: [PATCH 013/261] chore: add license checker (#14)

* chore: add license checker

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix license checker config

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/license_check.yml | 19 +++++++++++++++++++
 .licenserc.yaml                     |  9 +++++++++
 foyer-bench/src/analyze.rs          | 14 ++++++++++++++
 foyer-bench/src/rate.rs             | 14 ++++++++++++++
 foyer-bench/src/utils.rs            | 20 +++++++++++++++++++-
 5 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/license_check.yml
 create mode 100644 .licenserc.yaml

diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml
new file mode 100644
index 00000000..762f4294
--- /dev/null
+++ b/.github/workflows/license_check.yml
@@ -0,0 +1,19 @@
+name: License Checker
+
+on:
+  push:
+    branches:
+    - main
+    - "forks/*"
+  pull_request:
+    branches: 
+    - main
+    - "v*.*.*-rc"
+jobs:
+  license-header-check:
+    runs-on: ubuntu-latest
+    name: license-header-check
+    steps:
+    - uses: actions/checkout@v3
+    - name: Check License Header
+      uses: apache/skywalking-eyes/header@df70871af1a8109c9a5b1dc824faaf65246c5236
diff --git a/.licenserc.yaml b/.licenserc.yaml
new file mode 100644
index 00000000..33c44e0d
--- /dev/null
+++ b/.licenserc.yaml
@@ -0,0 +1,9 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: MrCroxx
+
+  paths:
+    - "**/*.rs"
+
+  comment: on-failure
diff --git a/foyer-bench/src/analyze.rs b/foyer-bench/src/analyze.rs
index 4b4be02e..e07a0681 100644
--- a/foyer-bench/src/analyze.rs
+++ b/foyer-bench/src/analyze.rs
@@ -1,3 +1,17 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
 // Copyright 2023 RisingWave Labs
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/foyer-bench/src/rate.rs b/foyer-bench/src/rate.rs
index a2d0853a..82164aab 100644
--- a/foyer-bench/src/rate.rs
+++ b/foyer-bench/src/rate.rs
@@ -1,3 +1,17 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
 // Copyright 2023 RisingWave Labs
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/foyer-bench/src/utils.rs b/foyer-bench/src/utils.rs
index 0b2a7ffe..620cd16e 100644
--- a/foyer-bench/src/utils.rs
+++ b/foyer-bench/src/utils.rs
@@ -1,3 +1,17 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
 // Copyright 2023 RisingWave Labs
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,8 +31,8 @@ use std::path::{Path, PathBuf};
 use itertools::Itertools;
 use nix::fcntl::readlink;
 use nix::sys::stat::stat;
-use nix::sys::statfs::{statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC};
 
+#[allow(unused)]
 #[derive(PartialEq, Clone, Copy, Debug)]
 pub enum FsType {
     Xfs,
@@ -28,9 +42,13 @@ pub enum FsType {
     Others,
 }
 
+#[allow(unused)]
 pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
     #[cfg(target_os = "linux")]
     {
+        use nix::sys::statfs::{
+            statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC,
+        };
         let fs_stat = statfs(path.as_ref()).unwrap();
         match fs_stat.filesystem_type() {
             XFS_SUPER_MAGIC => FsType::Xfs,

From e38ad6ad7f81b16bc20bcfbbc77e01976e3ec301 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 29 May 2023 15:04:35 +0800
Subject: [PATCH 014/261] chore: wrapper cache type, reorg utils, add metrics
 framework (#15)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock                                    | 62 +++++++++++++++++++
 Cargo.toml                                    |  2 +-
 foyer-bench/src/main.rs                       | 17 +++--
 foyer-utils/Cargo.toml                        | 40 ++++++++++++
 .../store/utils.rs => foyer-utils/src/bits.rs |  0
 .../collections => foyer-utils/src}/dlist.rs  |  5 +-
 .../mod.rs => foyer-utils/src/lib.rs          |  3 +
 foyer/Cargo.toml                              |  2 +
 foyer/src/container.rs                        |  5 +-
 foyer/src/lib.rs                              | 50 +++++++++++++--
 foyer/src/metrics.rs                          | 61 ++++++++++++++++++
 foyer/src/policies/lru.rs                     |  3 +-
 foyer/src/policies/tinylfu.rs                 |  3 +-
 foyer/src/store/mod.rs                        | 14 ++---
 14 files changed, 232 insertions(+), 35 deletions(-)
 create mode 100644 foyer-utils/Cargo.toml
 rename foyer/src/store/utils.rs => foyer-utils/src/bits.rs (100%)
 rename {foyer/src/collections => foyer-utils/src}/dlist.rs (99%)
 rename foyer/src/collections/mod.rs => foyer-utils/src/lib.rs (93%)
 create mode 100644 foyer/src/metrics.rs

diff --git a/Cargo.lock b/Cargo.lock
index cdae3f61..649b2b31 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -295,6 +295,12 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "foyer"
 version = "0.1.0"
@@ -305,6 +311,7 @@ dependencies = [
  "clap",
  "cmsketch",
  "crossbeam",
+ "foyer-utils",
  "futures",
  "hdrhistogram",
  "itertools",
@@ -313,6 +320,7 @@ dependencies = [
  "nix",
  "parking_lot",
  "paste",
+ "prometheus",
  "rand",
  "rand_mt",
  "tempfile",
@@ -340,6 +348,33 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "foyer-utils"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "bytesize",
+ "clap",
+ "cmsketch",
+ "crossbeam",
+ "futures",
+ "hdrhistogram",
+ "itertools",
+ "libc",
+ "memoffset 0.8.0",
+ "nix",
+ "parking_lot",
+ "paste",
+ "prometheus",
+ "rand",
+ "rand_mt",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "twox-hash",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.28"
@@ -516,6 +551,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
 [[package]]
 name = "libc"
 version = "0.2.144"
@@ -703,6 +744,27 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prometheus"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
+dependencies = [
+ "cfg-if",
+ "fnv",
+ "lazy_static",
+ "memchr",
+ "parking_lot",
+ "protobuf",
+ "thiserror",
+]
+
+[[package]]
+name = "protobuf"
+version = "2.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
+
 [[package]]
 name = "quote"
 version = "1.0.27"
diff --git a/Cargo.toml b/Cargo.toml
index 628cc248..d4e9a2eb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [workspace]
 
-members = ["foyer", "foyer-bench"]
+members = ["foyer", "foyer-bench", "foyer-utils"]
 
 [patch.crates-io]
 # cmsketch = { path = "../cmsketch-rs" }
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 3d2a300a..d5263ed3 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -27,11 +27,10 @@ use std::time::{Duration, Instant};
 use analyze::{analyze, monitor, Metrics};
 use clap::Parser;
 
-use foyer::container::{Config, Container};
-
-use foyer::store::read_only_file_store::{Config as ReadOnlyFileStoreConfig, ReadOnlyFileStore};
-
-use foyer::policies::tinylfu::{Config as TinyLfuConfig, Handle, TinyLfu};
+use foyer::{
+    ReadOnlyFileStoreConfig, TinyLfuConfig, TinyLfuReadOnlyFileStoreCache,
+    TinyLfuReadOnlyFileStoreCacheConfig,
+};
 use futures::future::join_all;
 use itertools::Itertools;
 use rand::rngs::StdRng;
@@ -109,9 +108,7 @@ impl Args {
         assert!(self.pools.is_power_of_two());
     }
 }
-
-type TContainer =
-    Container<u64, TinyLfu<u64>, Handle<u64>, Vec<u8>, ReadOnlyFileStore<u64, Vec<u8>>>;
+type TContainer = TinyLfuReadOnlyFileStoreCache<u64, Vec<u8>>;
 
 fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
@@ -159,7 +156,7 @@ async fn main() {
         tiny_lru_capacity_ratio: 0.01,
     };
 
-    let config: Config<u64, TinyLfu<_>, _, ReadOnlyFileStore<_, Vec<u8>>> = Config {
+    let config = TinyLfuReadOnlyFileStoreCacheConfig {
         capacity: args.capacity * 1024 * 1024,
         pool_count_bits: (args.pools as f64).log2() as usize,
         policy_config,
@@ -168,7 +165,7 @@ async fn main() {
 
     println!("{:#?}", config);
 
-    let container = Container::open(config).await.unwrap();
+    let container = TinyLfuReadOnlyFileStoreCache::open(config).await.unwrap();
     let container = Arc::new(container);
 
     let (iostat_stop_tx, iostat_stop_rx) = oneshot::channel();
diff --git a/foyer-utils/Cargo.toml b/foyer-utils/Cargo.toml
new file mode 100644
index 00000000..e1bfd5b7
--- /dev/null
+++ b/foyer-utils/Cargo.toml
@@ -0,0 +1,40 @@
+[package]
+name = "foyer-utils"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1"
+bytes = "1"
+cmsketch = "0.1"
+crossbeam = "0.8"
+futures = "0.3"
+itertools = "0.10.5"
+libc = "0.2"
+memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+paste = "1.0"
+prometheus = "0.13"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+twox-hash = "1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+hdrhistogram = "7"
+rand = "0.8.5"
+rand_mt = "4.2.1"
+tempfile = "3"
diff --git a/foyer/src/store/utils.rs b/foyer-utils/src/bits.rs
similarity index 100%
rename from foyer/src/store/utils.rs
rename to foyer-utils/src/bits.rs
diff --git a/foyer/src/collections/dlist.rs b/foyer-utils/src/dlist.rs
similarity index 99%
rename from foyer/src/collections/dlist.rs
rename to foyer-utils/src/dlist.rs
index 4e5a2ce8..a134ca84 100644
--- a/foyer/src/collections/dlist.rs
+++ b/foyer-utils/src/dlist.rs
@@ -33,13 +33,14 @@ pub trait Adapter<E> {
     fn el2en(_: NonNull<E>) -> NonNull<Entry>;
 }
 
+#[macro_export]
 macro_rules! intrusive_dlist {
     (
         $element:ident $(< $( $lt:tt $( : $clt:tt $(+ $dlt:tt)* )? ),+ >)?, $entry:ident, $adapter:ident
     ) => {
         struct $adapter;
 
-        impl $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? $crate::collections::dlist::Adapter<$element $(< $( $lt ),+ >)?> for $adapter  {
+        impl $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? $crate::dlist::Adapter<$element $(< $( $lt ),+ >)?> for $adapter  {
             fn en2el(entry: std::ptr::NonNull<Entry>) -> std::ptr::NonNull<$element $(< $( $lt ),+ >)?> {
                 unsafe {
                     let ptr = (entry.as_ptr() as *mut u8)
@@ -61,8 +62,6 @@ macro_rules! intrusive_dlist {
     };
 }
 
-pub(crate) use intrusive_dlist;
-
 /// TODO: write docs
 #[derive(Debug)]
 pub struct DList<E, A: Adapter<E>> {
diff --git a/foyer/src/collections/mod.rs b/foyer-utils/src/lib.rs
similarity index 93%
rename from foyer/src/collections/mod.rs
rename to foyer-utils/src/lib.rs
index 4fe9e427..2d58d97e 100644
--- a/foyer/src/collections/mod.rs
+++ b/foyer-utils/src/lib.rs
@@ -12,4 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+#![feature(trait_alias)]
+
+pub mod bits;
 pub mod dlist;
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 69bdec42..720e6ca6 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -12,6 +12,7 @@ async-trait = "0.1"
 bytes = "1"
 cmsketch = "0.1"
 crossbeam = "0.8"
+foyer-utils = { path = "../foyer-utils" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
@@ -19,6 +20,7 @@ memoffset = "0.8"
 nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
+prometheus = "0.13"
 thiserror = "1"
 tokio = { version = "1", features = [
     "rt",
diff --git a/foyer/src/container.rs b/foyer/src/container.rs
index eff0be4b..786ac6e4 100644
--- a/foyer/src/container.rs
+++ b/foyer/src/container.rs
@@ -97,7 +97,7 @@ where
             .into_iter()
             .enumerate()
             .map(|(id, store)| Pool {
-                id,
+                _id: id,
                 policy: P::new(config.policy_config.clone()),
                 capacity,
                 size: 0,
@@ -175,8 +175,7 @@ where
     D: Data,
     S: Store<I = I, D = D>,
 {
-    #[allow(unused)]
-    id: usize,
+    _id: usize,
 
     policy: P,
 
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 47fd517f..3ef71eee 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -21,10 +21,10 @@ pub use policies::Policy;
 
 use paste::paste;
 
-pub mod collections;
-pub mod container;
-pub mod policies;
-pub mod store;
+mod container;
+mod metrics;
+mod policies;
+mod store;
 
 pub trait Weight {
     fn weight(&self) -> usize;
@@ -87,9 +87,47 @@ pub struct WrappedNonNull<T>(pub NonNull<T>);
 unsafe impl<T> Send for WrappedNonNull<T> {}
 unsafe impl<T> Sync for WrappedNonNull<T> {}
 
-#[cfg(test)]
+// TODO(MrCroxx): Add global error type.
+pub type Error = store::error::Error;
+
+pub type TinyLfuReadOnlyFileStoreCache<I, D> = container::Container<
+    I,
+    policies::tinylfu::TinyLfu<I>,
+    policies::tinylfu::Handle<I>,
+    D,
+    store::read_only_file_store::ReadOnlyFileStore<I, D>,
+>;
+
+pub type TinyLfuReadOnlyFileStoreCacheConfig<I, D> = container::Config<
+    I,
+    policies::tinylfu::TinyLfu<I>,
+    policies::tinylfu::Handle<I>,
+    store::read_only_file_store::ReadOnlyFileStore<I, D>,
+>;
+
+pub type LruReadOnlyFileStoreCache<I, D> = container::Container<
+    I,
+    policies::lru::Lru<I>,
+    policies::lru::Handle<I>,
+    D,
+    store::read_only_file_store::ReadOnlyFileStore<I, D>,
+>;
+
+pub type LruReadOnlyFileStoreCacheConfig<I, D> = container::Config<
+    I,
+    policies::lru::Lru<I>,
+    policies::lru::Handle<I>,
+    store::read_only_file_store::ReadOnlyFileStore<I, D>,
+>;
+
+pub use metrics::Metrics;
+
+pub use policies::lru::Config as LruConfig;
+pub use policies::tinylfu::Config as TinyLfuConfig;
+pub use store::read_only_file_store::Config as ReadOnlyFileStoreConfig;
 
-pub mod tests {
+#[cfg(test)]
 
+mod tests {
     pub fn is_send_sync_static<T: Send + Sync + 'static>() {}
 }
diff --git a/foyer/src/metrics.rs b/foyer/src/metrics.rs
new file mode 100644
index 00000000..aa84a4cb
--- /dev/null
+++ b/foyer/src/metrics.rs
@@ -0,0 +1,61 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use prometheus::{
+    register_counter_vec_with_registry, register_histogram_vec_with_registry,
+    register_int_counter_with_registry, CounterVec, HistogramVec, IntCounter, Registry,
+};
+
+pub struct Metrics {
+    pub miss: IntCounter,
+
+    pub latency: HistogramVec,
+    pub bytes: CounterVec,
+}
+
+impl Default for Metrics {
+    fn default() -> Self {
+        Self::new(Registry::new())
+    }
+}
+
+impl Metrics {
+    pub fn new(registry: Registry) -> Self {
+        let miss =
+            register_int_counter_with_registry!("foyer_cache_miss", "file cache miss", registry)
+                .unwrap();
+
+        let latency = register_histogram_vec_with_registry!(
+            "foyer_latency",
+            "foyer latency",
+            &["op"],
+            vec![
+                0.0001, 0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75,
+                1.0
+            ],
+            registry
+        )
+        .unwrap();
+
+        let bytes =
+            register_counter_vec_with_registry!("foyer_bytes", "foyer bytes", &["op"], registry)
+                .unwrap();
+
+        Self {
+            miss,
+            latency,
+            bytes,
+        }
+    }
+}
diff --git a/foyer/src/policies/lru.rs b/foyer/src/policies/lru.rs
index 50a53353..ebac9999 100644
--- a/foyer/src/policies/lru.rs
+++ b/foyer/src/policies/lru.rs
@@ -29,7 +29,8 @@
 use std::ptr::NonNull;
 use std::time::SystemTime;
 
-use crate::collections::dlist::{intrusive_dlist, DList, Entry, Iter};
+use foyer_utils::dlist::{DList, Entry, Iter};
+use foyer_utils::intrusive_dlist;
 
 use super::Index;
 
diff --git a/foyer/src/policies/tinylfu.rs b/foyer/src/policies/tinylfu.rs
index a758a997..e8e44144 100644
--- a/foyer/src/policies/tinylfu.rs
+++ b/foyer/src/policies/tinylfu.rs
@@ -33,7 +33,8 @@ use std::time::SystemTime;
 use cmsketch::CMSketchUsize;
 use twox_hash::XxHash64;
 
-use crate::collections::dlist::{intrusive_dlist, DList, Entry, Iter};
+use foyer_utils::dlist::{DList, Entry, Iter};
+use foyer_utils::intrusive_dlist;
 
 use super::Index;
 
diff --git a/foyer/src/store/mod.rs b/foyer/src/store/mod.rs
index cbebbaa4..e5d46e20 100644
--- a/foyer/src/store/mod.rs
+++ b/foyer/src/store/mod.rs
@@ -13,9 +13,9 @@
 //  limitations under the License.
 
 pub mod error;
+#[allow(dead_code)]
 pub mod file;
 pub mod read_only_file_store;
-pub mod utils;
 
 use crate::{Data, Index};
 use async_trait::async_trait;
@@ -62,21 +62,15 @@ pub mod tests {
 
     #[derive(Clone, Debug, Default)]
     pub struct MemoryStore<I: Index, D: Data + Clone> {
-        pool: usize,
         inner: Arc<RwLock<HashMap<I, D>>>,
     }
 
     impl<I: Index, D: Data + Clone> MemoryStore<I, D> {
-        pub fn new(pool: usize) -> Self {
+        pub fn new() -> Self {
             Self {
-                pool,
                 inner: Arc::new(RwLock::new(HashMap::default())),
             }
         }
-
-        pub fn pool(&self) -> usize {
-            self.pool
-        }
     }
 
     #[async_trait]
@@ -87,8 +81,8 @@ pub mod tests {
 
         type C = ();
 
-        async fn open(pool: usize, _: Self::C) -> Result<Self> {
-            Ok(Self::new(pool))
+        async fn open(_pool: usize, _: Self::C) -> Result<Self> {
+            Ok(Self::new())
         }
 
         async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {

From f44aec15c6e46971a4f4dba7eec5b5bfa366ccc2 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 29 May 2023 16:24:52 +0800
Subject: [PATCH 015/261] fix: fix read only store recovery, impl random drop
 (#16)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer/Cargo.toml                        |   2 +-
 foyer/src/store/read_only_file_store.rs | 115 +++++++++++++++++++++---
 2 files changed, 102 insertions(+), 15 deletions(-)

diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 720e6ca6..bbc3834d 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -21,6 +21,7 @@ nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
 prometheus = "0.13"
+rand = "0.8.5"
 thiserror = "1"
 tokio = { version = "1", features = [
     "rt",
@@ -36,6 +37,5 @@ twox-hash = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 hdrhistogram = "7"
-rand = "0.8.5"
 rand_mt = "4.2.1"
 tempfile = "3"
diff --git a/foyer/src/store/read_only_file_store.rs b/foyer/src/store/read_only_file_store.rs
index 8730fbdf..c75c5ed8 100644
--- a/foyer/src/store/read_only_file_store.rs
+++ b/foyer/src/store/read_only_file_store.rs
@@ -23,7 +23,9 @@ use std::{collections::HashMap, marker::PhantomData};
 
 use async_trait::async_trait;
 
+use bytes::{Buf, BufMut};
 use itertools::Itertools;
+use rand::{thread_rng, Rng};
 use tokio::sync::{RwLock, RwLockWriteGuard};
 
 use crate::{Data, Index};
@@ -139,19 +141,18 @@ where
             let dir = config.dir.clone();
             move || {
                 create_dir_all(&dir)?;
-                let ids: Vec<FileId> = read_dir(dir)?
+
+                let mut ids: Vec<FileId> = read_dir(dir)?
                     .map(|entry| entry.unwrap())
                     .filter(|entry| {
-                        entry
-                            .file_name()
-                            .to_string_lossy()
-                            .is_prefix_of(META_FILE_PREFIX)
+                        META_FILE_PREFIX.is_prefix_of(&entry.file_name().to_string_lossy())
                     })
                     .map(|entry| {
                         entry.file_name().to_string_lossy()[META_FILE_PREFIX.len()..].to_string()
                     })
                     .map(|s| s.parse().unwrap())
                     .collect_vec();
+                ids.sort();
                 Ok(ids)
             }
         })
@@ -187,11 +188,20 @@ where
 
     #[allow(clippy::uninit_vec)]
     async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {
-        let buf = data.into();
-        let len = buf.len();
-
         // append cache file and meta file
         let (fid, sid, location) = {
+            // randomly drop if size exceeds the threshold
+            if self.size.load(Ordering::Relaxed) as f64
+                >= self.config.capacity as f64 * self.config.trigger_random_drop_ratio
+            {
+                let mut rng = thread_rng();
+                if rng.gen_range(0.0..1.0) < self.config.random_drop_ratio {
+                    return Ok(());
+                }
+            }
+
+            let buf = data.into();
+
             let files = self.files.read().await;
 
             let fid = files.active.fid;
@@ -199,9 +209,14 @@ where
             let location = files.active.cache_file.append(buf).await?;
 
             let mut buf = Vec::with_capacity(Self::meta_entry_size());
+
+            let tags = Tags { valid: true };
+
             unsafe { buf.set_len(Self::meta_entry_size()) };
-            index.write(&mut buf[..]);
-            location.write(&mut buf[I::size()..]);
+            tags.write(&mut buf[..]);
+            index.write(&mut buf[Tags::size()..]);
+            location.write(&mut buf[Tags::size() + I::size()..]);
+
             let Location {
                 offset: meta_offset,
                 len: _,
@@ -219,7 +234,8 @@ where
             drop(indices);
         }
 
-        self.size.fetch_add(len, Ordering::Relaxed);
+        self.size
+            .fetch_add(location.len as usize, Ordering::Relaxed);
 
         if active_file_size >= self.config.max_file_size {
             let files = self.files.write().await;
@@ -434,8 +450,12 @@ where
         let buf = meta.read(0, size).await?;
         for sid in 0..slots {
             let slice = &buf[sid * Self::meta_entry_size()..(sid + 1) * Self::meta_entry_size()];
-            let index = I::read(slice);
-            let location = Location::read(&slice[I::size()..]);
+            let tags = Tags::read(slice);
+            if !tags.valid {
+                continue;
+            }
+            let index = I::read(&slice[Tags::size()..]);
+            let location = Location::read(&slice[Tags::size() + I::size()..]);
             indices.insert(index, (fid, sid as SlotId, location));
         }
         Ok(())
@@ -450,7 +470,29 @@ where
     }
 
     fn meta_entry_size() -> usize {
-        I::size() + Location::size()
+        I::size() + Location::size() + Tags::size()
+    }
+}
+
+struct Tags {
+    valid: bool,
+}
+
+impl Tags {
+    fn size() -> usize {
+        1
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        let mut val = 0;
+        val |= self.valid as u8;
+        buf.put_u8(val);
+    }
+
+    fn read(mut buf: &[u8]) -> Self {
+        let val = buf.get_u8();
+        let valid = (val & 1) != 0;
+        Self { valid }
     }
 }
 
@@ -514,4 +556,49 @@ mod tests {
 
         drop(dir);
     }
+
+    #[tokio::test]
+    async fn test_read_only_file_store_recovery() {
+        let dir = tempdir().unwrap();
+
+        let config = Config {
+            dir: dir.path().to_owned(),
+            max_file_size: 4 * 1024,
+            capacity: 16 * 1024,
+            trigger_reclaim_garbage_ratio: 0.0, // disabled
+            trigger_reclaim_capacity_ratio: 0.75,
+            trigger_random_drop_ratio: 0.0, // disabled
+            random_drop_ratio: 0.0,         // disabled
+        };
+
+        let store: ReadOnlyFileStore<u64, Vec<u8>> =
+            ReadOnlyFileStore::open(0, config.clone()).await.unwrap();
+
+        for i in 0..20 {
+            store.store(i, data(i as u8, 1024)).await.unwrap();
+        }
+
+        assert_eq!(store.files.read().await.frozens.len(), 2);
+        for i in 0..12 {
+            assert_eq!(store.load(&i).await.unwrap(), None);
+        }
+        for i in 12..20 {
+            assert_eq!(store.load(&i).await.unwrap(), Some(data(i as u8, 1024)));
+        }
+
+        drop(store);
+
+        let store: ReadOnlyFileStore<u64, Vec<u8>> =
+            ReadOnlyFileStore::open(0, config).await.unwrap();
+
+        assert_eq!(store.files.read().await.frozens.len(), 3);
+        for i in 0..12 {
+            assert_eq!(store.load(&i).await.unwrap(), None);
+        }
+        for i in 12..20 {
+            assert_eq!(store.load(&i).await.unwrap(), Some(data(i as u8, 1024)));
+        }
+
+        drop(dir);
+    }
 }

From 8e80c6c240ff39faff7cf387d902de2a5e3609b7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 29 May 2023 17:25:58 +0800
Subject: [PATCH 016/261] feat: add metrics (#17)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer/src/container.rs                  | 34 +++++++++++--
 foyer/src/metrics.rs                    | 68 +++++++++++++++++++++----
 foyer/src/store/mod.rs                  |  8 +--
 foyer/src/store/read_only_file_store.rs | 49 ++++++++++++++----
 4 files changed, 133 insertions(+), 26 deletions(-)

diff --git a/foyer/src/container.rs b/foyer/src/container.rs
index 786ac6e4..583080a6 100644
--- a/foyer/src/container.rs
+++ b/foyer/src/container.rs
@@ -16,6 +16,7 @@ use std::collections::BTreeMap;
 use std::hash::Hasher;
 
 use std::ptr::NonNull;
+use std::sync::Arc;
 
 use futures::future::try_join_all;
 use itertools::Itertools;
@@ -24,7 +25,7 @@ use twox_hash::XxHash64;
 
 use crate::policies::{Handle, Policy};
 use crate::store::Store;
-use crate::{Data, Index, WrappedNonNull};
+use crate::{Data, Index, Metrics, WrappedNonNull};
 
 // TODO(MrCroxx): wrap own result type
 use crate::store::error::Result;
@@ -73,6 +74,8 @@ where
 {
     pool_count_bits: usize,
     pools: Vec<Mutex<Pool<I, P, H, D, S>>>,
+
+    metrics: Arc<Metrics>,
 }
 
 impl<I, P, H, D, S> Container<I, P, H, D, S>
@@ -85,11 +88,20 @@ where
     S: Store<I = I, D = D>,
 {
     pub async fn open(config: Config<I, P, H, S>) -> Result<Self> {
+        Self::open_with_registry(config, prometheus::Registry::new()).await
+    }
+
+    pub async fn open_with_registry(
+        config: Config<I, P, H, S>,
+        registry: prometheus::Registry,
+    ) -> Result<Self> {
         let pool_count = 1 << config.pool_count_bits;
         let capacity = config.capacity >> config.pool_count_bits;
 
+        let metrics = Arc::new(Metrics::new(registry));
+
         let stores = (0..pool_count)
-            .map(|pool| S::open(pool, config.store_config.clone()))
+            .map(|pool| S::open(pool, config.store_config.clone(), metrics.clone()))
             .collect_vec();
         let stores = try_join_all(stores).await?;
 
@@ -103,6 +115,7 @@ where
                 size: 0,
                 handles: BTreeMap::new(),
                 store,
+                _metrics: metrics.clone(),
             })
             .map(Mutex::new)
             .collect_vec();
@@ -110,10 +123,13 @@ where
         Ok(Self {
             pool_count_bits: config.pool_count_bits,
             pools,
+            metrics,
         })
     }
 
     pub async fn insert(&self, index: I, data: D) -> Result<bool> {
+        let _timer = self.metrics.latency_insert.start_timer();
+
         let mut pool = self.pool(&index).await;
 
         if pool.handles.get(&index).is_some() {
@@ -131,6 +147,8 @@ where
     }
 
     pub async fn remove(&self, index: &I) -> Result<bool> {
+        let _timer = self.metrics.latency_remove.start_timer();
+
         let mut pool = self.pool(index).await;
 
         if pool.handles.get(index).is_none() {
@@ -144,9 +162,17 @@ where
     }
 
     pub async fn get(&self, index: &I) -> Result<Option<D>> {
+        let _timer = self.metrics.latency_get.start_timer();
+
         let mut pool = self.pool(index).await;
 
-        pool.get(index).await
+        let res = pool.get(index).await?;
+
+        if res.is_none() {
+            self.metrics.miss.inc();
+        }
+
+        Ok(res)
     }
 
     // TODO(MrCroxx): optimize this
@@ -186,6 +212,8 @@ where
     handles: BTreeMap<I, PoolHandle<I, H>>,
 
     store: S,
+
+    _metrics: Arc<Metrics>,
 }
 
 impl<I, P, H, D, S> Pool<I, P, H, D, S>
diff --git a/foyer/src/metrics.rs b/foyer/src/metrics.rs
index aa84a4cb..09f92107 100644
--- a/foyer/src/metrics.rs
+++ b/foyer/src/metrics.rs
@@ -13,15 +13,30 @@
 //  limitations under the License.
 
 use prometheus::{
-    register_counter_vec_with_registry, register_histogram_vec_with_registry,
-    register_int_counter_with_registry, CounterVec, HistogramVec, IntCounter, Registry,
+    register_counter_vec_with_registry, register_gauge_with_registry,
+    register_histogram_vec_with_registry, register_int_counter_with_registry, Counter, CounterVec,
+    Gauge, Histogram, HistogramVec, IntCounter, Registry,
 };
 
 pub struct Metrics {
+    pub latency_insert: Histogram,
+    pub latency_get: Histogram,
+    pub latency_remove: Histogram,
+
+    pub latency_store: Histogram,
+    pub latency_load: Histogram,
+    pub latency_delete: Histogram,
+
+    pub bytes_store: Counter,
+    pub bytes_load: Counter,
+    pub bytes_delete: Counter,
+
+    pub cache_data_size: Gauge,
+
     pub miss: IntCounter,
 
-    pub latency: HistogramVec,
-    pub bytes: CounterVec,
+    _latency: HistogramVec,
+    _bytes: CounterVec,
 }
 
 impl Default for Metrics {
@@ -32,10 +47,6 @@ impl Default for Metrics {
 
 impl Metrics {
     pub fn new(registry: Registry) -> Self {
-        let miss =
-            register_int_counter_with_registry!("foyer_cache_miss", "file cache miss", registry)
-                .unwrap();
-
         let latency = register_histogram_vec_with_registry!(
             "foyer_latency",
             "foyer latency",
@@ -52,10 +63,47 @@ impl Metrics {
             register_counter_vec_with_registry!("foyer_bytes", "foyer bytes", &["op"], registry)
                 .unwrap();
 
+        let latency_insert = latency.with_label_values(&["insert"]);
+        let latency_get = latency.with_label_values(&["get"]);
+        let latency_remove = latency.with_label_values(&["remove"]);
+
+        let latency_store = latency.with_label_values(&["store"]);
+        let latency_load = latency.with_label_values(&["load"]);
+        let latency_delete = latency.with_label_values(&["delete"]);
+
+        let bytes_store = bytes.with_label_values(&["store"]);
+        let bytes_load = bytes.with_label_values(&["load"]);
+        let bytes_delete = bytes.with_label_values(&["delete"]);
+
+        let miss =
+            register_int_counter_with_registry!("foyer_cache_miss", "foyer cache miss", registry)
+                .unwrap();
+        let cache_data_size = register_gauge_with_registry!(
+            "foyer_cache_data_size",
+            "foyer cache data size",
+            registry
+        )
+        .unwrap();
+
         Self {
+            latency_insert,
+            latency_get,
+            latency_remove,
+
+            latency_store,
+            latency_load,
+            latency_delete,
+
+            bytes_store,
+            bytes_load,
+            bytes_delete,
+
+            cache_data_size,
+
             miss,
-            latency,
-            bytes,
+
+            _latency: latency,
+            _bytes: bytes,
         }
     }
 }
diff --git a/foyer/src/store/mod.rs b/foyer/src/store/mod.rs
index e5d46e20..69a861f7 100644
--- a/foyer/src/store/mod.rs
+++ b/foyer/src/store/mod.rs
@@ -17,7 +17,9 @@ pub mod error;
 pub mod file;
 pub mod read_only_file_store;
 
-use crate::{Data, Index};
+use std::sync::Arc;
+
+use crate::{Data, Index, Metrics};
 use async_trait::async_trait;
 
 use error::Result;
@@ -28,7 +30,7 @@ pub trait Store: Send + Sync + Sized + 'static {
     type D: Data;
     type C: Send + Sync + Clone + std::fmt::Debug + 'static;
 
-    async fn open(pool: usize, config: Self::C) -> Result<Self>;
+    async fn open(pool: usize, config: Self::C, metrics: Arc<Metrics>) -> Result<Self>;
 
     async fn store(&self, index: Self::I, data: Self::D) -> Result<()>;
 
@@ -81,7 +83,7 @@ pub mod tests {
 
         type C = ();
 
-        async fn open(_pool: usize, _: Self::C) -> Result<Self> {
+        async fn open(_pool: usize, _: Self::C, _metrics: Arc<Metrics>) -> Result<Self> {
             Ok(Self::new())
         }
 
diff --git a/foyer/src/store/read_only_file_store.rs b/foyer/src/store/read_only_file_store.rs
index c75c5ed8..e68d72cf 100644
--- a/foyer/src/store/read_only_file_store.rs
+++ b/foyer/src/store/read_only_file_store.rs
@@ -28,7 +28,7 @@ use itertools::Itertools;
 use rand::{thread_rng, Rng};
 use tokio::sync::{RwLock, RwLockWriteGuard};
 
-use crate::{Data, Index};
+use crate::{Data, Index, Metrics};
 
 use super::error::Result;
 use super::file::{AppendableFile, Location, ReadableFile, WritableFile};
@@ -102,6 +102,8 @@ where
 
     size: Arc<AtomicUsize>,
 
+    metrics: Arc<Metrics>,
+
     _marker: PhantomData<D>,
 }
 
@@ -117,6 +119,7 @@ where
             indices: Arc::clone(&self.indices),
             files: Arc::clone(&self.files),
             size: Arc::clone(&self.size),
+            metrics: Arc::clone(&self.metrics),
             _marker: PhantomData,
         }
     }
@@ -134,7 +137,7 @@ where
 
     type C = Config;
 
-    async fn open(pool: usize, mut config: Self::C) -> Result<Self> {
+    async fn open(pool: usize, mut config: Self::C, metrics: Arc<Metrics>) -> Result<Self> {
         config.dir = config.dir.join(format!("{:04}", pool));
 
         let ids = asyncify({
@@ -182,12 +185,15 @@ where
             indices: Arc::new(RwLock::new(indices)),
             files: Arc::new(RwLock::new(files)),
             size: Arc::new(AtomicUsize::new(size)),
+            metrics,
             _marker: PhantomData,
         })
     }
 
     #[allow(clippy::uninit_vec)]
     async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {
+        let _timer = self.metrics.latency_store.start_timer();
+
         // append cache file and meta file
         let (fid, sid, location) = {
             // randomly drop if size exceeds the threshold
@@ -234,8 +240,13 @@ where
             drop(indices);
         }
 
-        self.size
-            .fetch_add(location.len as usize, Ordering::Relaxed);
+        let cache_data_size = self
+            .size
+            .fetch_add(location.len as usize, Ordering::Relaxed)
+            + location.len as usize;
+
+        self.metrics.bytes_store.inc_by(location.len as f64);
+        self.metrics.cache_data_size.set(cache_data_size as f64);
 
         if active_file_size >= self.config.max_file_size {
             let files = self.files.write().await;
@@ -251,6 +262,8 @@ where
     }
 
     async fn load(&self, index: &Self::I) -> Result<Option<Self::D>> {
+        let _timer = self.metrics.latency_load.start_timer();
+
         // TODO(MrCroxx): add bloom filters ?
         let (fid, _sid, location) = {
             let indices = self.indices.read().await;
@@ -283,12 +296,16 @@ where
             }
         };
 
+        self.metrics.bytes_load.inc_by(location.len as f64);
+
         self.maybe_trigger_reclaim().await?;
 
         Ok(Some(buf.into()))
     }
 
     async fn delete(&self, index: &Self::I) -> Result<()> {
+        let _timer = self.metrics.latency_delete.start_timer();
+
         let (fid, sid, location) = {
             let indices = self.indices.read().await;
             let (fid, sid, location) = match indices.get(index) {
@@ -318,8 +335,13 @@ where
             }
         }
 
-        self.size
-            .fetch_sub(location.len as usize, Ordering::Relaxed);
+        let cache_data_size = self
+            .size
+            .fetch_sub(location.len as usize, Ordering::Relaxed)
+            - location.len as usize;
+
+        self.metrics.bytes_delete.inc_by(location.len as f64);
+        self.metrics.cache_data_size.set(cache_data_size as f64);
 
         self.maybe_trigger_reclaim().await?;
 
@@ -376,7 +398,8 @@ where
             size as usize
         };
 
-        self.size.fetch_sub(size, Ordering::Relaxed);
+        let cache_data_size = self.size.fetch_sub(size, Ordering::Relaxed) - size;
+        self.metrics.cache_data_size.set(cache_data_size as f64);
 
         meta_file.reclaim().await?;
         cache_file.reclaim().await?;
@@ -520,7 +543,9 @@ mod tests {
         };
 
         let store: ReadOnlyFileStore<u64, Vec<u8>> =
-            ReadOnlyFileStore::open(0, config).await.unwrap();
+            ReadOnlyFileStore::open(0, config, Arc::new(Metrics::default()))
+                .await
+                .unwrap();
 
         store.store(1, data(1, 1024)).await.unwrap();
         assert_eq!(store.load(&1).await.unwrap(), Some(data(1, 1024)));
@@ -572,7 +597,9 @@ mod tests {
         };
 
         let store: ReadOnlyFileStore<u64, Vec<u8>> =
-            ReadOnlyFileStore::open(0, config.clone()).await.unwrap();
+            ReadOnlyFileStore::open(0, config.clone(), Arc::new(Metrics::default()))
+                .await
+                .unwrap();
 
         for i in 0..20 {
             store.store(i, data(i as u8, 1024)).await.unwrap();
@@ -589,7 +616,9 @@ mod tests {
         drop(store);
 
         let store: ReadOnlyFileStore<u64, Vec<u8>> =
-            ReadOnlyFileStore::open(0, config).await.unwrap();
+            ReadOnlyFileStore::open(0, config, Arc::new(Metrics::default()))
+                .await
+                .unwrap();
 
         assert_eq!(store.files.read().await.frozens.len(), 3);
         for i in 0..12 {

From 79610aabae80914e67e62956991725f07967f9d4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 1 Jun 2023 12:19:20 +0800
Subject: [PATCH 017/261] feat: add log support (#19)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock                              | 33 +++++++++++++++++++
 foyer/Cargo.toml                        |  1 +
 foyer/src/container.rs                  |  2 ++
 foyer/src/store/read_only_file_store.rs | 43 +++++++++++++++++--------
 4 files changed, 65 insertions(+), 14 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 649b2b31..6155477b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -326,6 +326,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "tokio",
+ "tracing",
  "twox-hash",
 ]
 
@@ -958,6 +959,38 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tracing"
+version = "0.1.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+dependencies = [
+ "cfg-if",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
+dependencies = [
+ "once_cell",
+]
+
 [[package]]
 name = "twox-hash"
 version = "1.6.3"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index bbc3834d..949daa8e 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -31,6 +31,7 @@ tokio = { version = "1", features = [
     "time",
     "signal",
 ] }
+tracing = "0.1"
 twox-hash = "1"
 
 [dev-dependencies]
diff --git a/foyer/src/container.rs b/foyer/src/container.rs
index 583080a6..e3a91b59 100644
--- a/foyer/src/container.rs
+++ b/foyer/src/container.rs
@@ -95,6 +95,8 @@ where
         config: Config<I, P, H, S>,
         registry: prometheus::Registry,
     ) -> Result<Self> {
+        tracing::info!("open foyer with config: \n{:#?}", config);
+
         let pool_count = 1 << config.pool_count_bits;
         let capacity = config.capacity >> config.pool_count_bits;
 
diff --git a/foyer/src/store/read_only_file_store.rs b/foyer/src/store/read_only_file_store.rs
index e68d72cf..c9258d88 100644
--- a/foyer/src/store/read_only_file_store.rs
+++ b/foyer/src/store/read_only_file_store.rs
@@ -286,13 +286,18 @@ where
                     .read(location.offset as u64, location.len as usize)
                     .await?
             } else {
-                files
-                    .frozens
-                    .get(&fid)
-                    .expect("frozen file not found")
-                    .cache_file
-                    .read(location.offset as u64, location.len as usize)
-                    .await?
+                match files.frozens.get(&fid) {
+                    Some(frozen) => {
+                        frozen
+                            .cache_file
+                            .read(location.offset as u64, location.len as usize)
+                            .await?
+                    }
+                    None => {
+                        tracing::error!("frozen file {} not found", fid);
+                        return Ok(None);
+                    }
+                }
             }
         };
 
@@ -325,13 +330,17 @@ where
                     .write(sid as u64 * Self::meta_entry_size() as u64, empty_entry)
                     .await?;
             } else {
-                files
-                    .frozens
-                    .get(&fid)
-                    .expect("frozen file not found")
-                    .meta_file
-                    .write(sid as u64 * Self::meta_entry_size() as u64, empty_entry)
-                    .await?;
+                match files.frozens.get(&fid) {
+                    Some(frozen) => {
+                        frozen
+                            .meta_file
+                            .write(sid as u64 * Self::meta_entry_size() as u64, empty_entry)
+                            .await?;
+                    }
+                    None => {
+                        tracing::error!("frozen file {} not found", fid);
+                    }
+                }
             }
         }
 
@@ -369,10 +378,14 @@ where
         // update frozen map
         files.frozens.insert(id, frozen);
 
+        tracing::info!("active file rotated: {} => {}", id, id + 1);
+
         Ok(())
     }
 
     async fn reclaim_frozen_file(&self, id: FileId) -> Result<()> {
+        tracing::info!("reclaiming frozen file {}", id);
+
         let (fid, meta_file, cache_file) = {
             let mut files = self.files.write().await;
 
@@ -404,6 +417,8 @@ where
         meta_file.reclaim().await?;
         cache_file.reclaim().await?;
 
+        tracing::info!("frozen file {} reclaimed", id);
+
         Ok(())
     }
 

From 23e2612ee80d2ce80b09f0a12b2d6849f391d114 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 8 Jun 2023 12:02:37 +0800
Subject: [PATCH 018/261] feat: add write stall config for read only file store
 (#20)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-bench/src/main.rs                 |  5 +++++
 foyer/src/store/read_only_file_store.rs | 27 ++++++++++++++++++-------
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index d5263ed3..4cc8922d 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -101,6 +101,10 @@ pub struct Args {
     /// read only file store: ratio of randomly dropped entries (0 ~ 1)
     #[arg(long, default_value_t = 0.0)]
     rofs_random_drop_ratio: f64,
+
+    /// read only file store: ratio of size to trigger write stall, every new entry to insert will be dropped (0 ~ 1)
+    #[arg(long, default_value_t = 0.0)]
+    rofs_write_stall_threshold_ratio: f64,
 }
 
 impl Args {
@@ -149,6 +153,7 @@ async fn main() {
         trigger_reclaim_capacity_ratio: args.rofs_trigger_reclaim_capacity_ratio,
         trigger_random_drop_ratio: args.rofs_trigger_random_drop_ratio,
         random_drop_ratio: args.rofs_random_drop_ratio,
+        write_stall_threshold_ratio: args.rofs_write_stall_threshold_ratio,
     };
 
     let policy_config = TinyLfuConfig {
diff --git a/foyer/src/store/read_only_file_store.rs b/foyer/src/store/read_only_file_store.rs
index c9258d88..452a1130 100644
--- a/foyer/src/store/read_only_file_store.rs
+++ b/foyer/src/store/read_only_file_store.rs
@@ -62,6 +62,11 @@ pub struct Config {
 
     /// ratio of randomly dropped entries
     pub random_drop_ratio: f64,
+
+    /// ratio of size to trigger write stall
+    ///
+    /// every new entry to insert will be dropped
+    pub write_stall_threshold_ratio: f64,
 }
 
 struct Frozen {
@@ -196,10 +201,16 @@ where
 
         // append cache file and meta file
         let (fid, sid, location) = {
-            // randomly drop if size exceeds the threshold
-            if self.size.load(Ordering::Relaxed) as f64
-                >= self.config.capacity as f64 * self.config.trigger_random_drop_ratio
+            let size_ratio = self.size.load(Ordering::Relaxed) as f64 / self.config.capacity as f64;
+            if self.config.write_stall_threshold_ratio > 0.0
+                && size_ratio >= self.config.write_stall_threshold_ratio
+            {
+                // write stall
+                return Ok(());
+            } else if self.config.trigger_random_drop_ratio > 0.0
+                && size_ratio >= self.config.trigger_random_drop_ratio
             {
+                // random drop
                 let mut rng = thread_rng();
                 if rng.gen_range(0.0..1.0) < self.config.random_drop_ratio {
                     return Ok(());
@@ -553,8 +564,9 @@ mod tests {
             capacity: 16 * 1024,
             trigger_reclaim_garbage_ratio: 0.0, // disabled
             trigger_reclaim_capacity_ratio: 0.75,
-            trigger_random_drop_ratio: 0.0, // disabled
-            random_drop_ratio: 0.0,         // disabled
+            trigger_random_drop_ratio: 0.0,   // disabled
+            random_drop_ratio: 0.0,           // disabled
+            write_stall_threshold_ratio: 0.0, // disabled
         };
 
         let store: ReadOnlyFileStore<u64, Vec<u8>> =
@@ -607,8 +619,9 @@ mod tests {
             capacity: 16 * 1024,
             trigger_reclaim_garbage_ratio: 0.0, // disabled
             trigger_reclaim_capacity_ratio: 0.75,
-            trigger_random_drop_ratio: 0.0, // disabled
-            random_drop_ratio: 0.0,         // disabled
+            trigger_random_drop_ratio: 0.0,   // disabled
+            random_drop_ratio: 0.0,           // disabled
+            write_stall_threshold_ratio: 0.0, // disabled
         };
 
         let store: ReadOnlyFileStore<u64, Vec<u8>> =

From 506f3efd265ea69dd67cd836486ff4d4b4b5c48e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 28 Jun 2023 12:06:39 +0800
Subject: [PATCH 019/261] feat: introduce instrusive indexers and collections
 (#21)

* feat: introduce instrusive indexers and collections

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock                                    |  90 +++
 Cargo.toml                                    |   9 +-
 foyer-common/Cargo.toml                       |  43 ++
 foyer-common/src/lib.rs                       | 102 +++
 foyer-intrusive/Cargo.toml                    |  42 ++
 foyer-intrusive/README.md                     |   8 +
 foyer-intrusive/src/collections/dlist.rs      | 513 +++++++++++++++
 foyer-intrusive/src/collections/mod.rs        |  15 +
 foyer-intrusive/src/core/adapter.rs           | 269 ++++++++
 foyer-intrusive/src/core/mod.rs               |  16 +
 foyer-intrusive/src/core/pointer.rs           | 565 ++++++++++++++++
 foyer-intrusive/src/eviction/lfu.rs           | 618 ++++++++++++++++++
 foyer-intrusive/src/eviction/lru.rs           | 446 +++++++++++++
 foyer-intrusive/src/eviction/mod.rs           |  57 ++
 foyer-intrusive/src/lib.rs                    |  56 ++
 foyer-policy/Cargo.toml                       |  42 ++
 .../src/eviction}/lru.rs                      |  80 ++-
 .../src/eviction}/mod.rs                      |  15 +-
 .../src/eviction}/tinylfu.rs                  |  98 ++-
 foyer-policy/src/lib.rs                       |  30 +
 foyer-policy/src/reinsertion/mod.rs           |  29 +
 foyer-utils/src/dlist.rs                      |  10 +-
 foyer-utils/src/lib.rs                        |   1 +
 foyer-utils/src/queue.rs                      |  64 ++
 foyer/Cargo.toml                              |   2 +
 foyer/src/container.rs                        |  36 +-
 foyer/src/lib.rs                              |  23 +-
 foyer/src/store/error.rs                      |   4 -
 28 files changed, 3145 insertions(+), 138 deletions(-)
 create mode 100644 foyer-common/Cargo.toml
 create mode 100644 foyer-common/src/lib.rs
 create mode 100644 foyer-intrusive/Cargo.toml
 create mode 100644 foyer-intrusive/README.md
 create mode 100644 foyer-intrusive/src/collections/dlist.rs
 create mode 100644 foyer-intrusive/src/collections/mod.rs
 create mode 100644 foyer-intrusive/src/core/adapter.rs
 create mode 100644 foyer-intrusive/src/core/mod.rs
 create mode 100644 foyer-intrusive/src/core/pointer.rs
 create mode 100644 foyer-intrusive/src/eviction/lfu.rs
 create mode 100644 foyer-intrusive/src/eviction/lru.rs
 create mode 100644 foyer-intrusive/src/eviction/mod.rs
 create mode 100644 foyer-intrusive/src/lib.rs
 create mode 100644 foyer-policy/Cargo.toml
 rename {foyer/src/policies => foyer-policy/src/eviction}/lru.rs (83%)
 rename {foyer/src/policies => foyer-policy/src/eviction}/mod.rs (86%)
 rename {foyer/src/policies => foyer-policy/src/eviction}/tinylfu.rs (86%)
 create mode 100644 foyer-policy/src/lib.rs
 create mode 100644 foyer-policy/src/reinsertion/mod.rs
 create mode 100644 foyer-utils/src/queue.rs

diff --git a/Cargo.lock b/Cargo.lock
index 6155477b..77800303 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -311,6 +311,8 @@ dependencies = [
  "clap",
  "cmsketch",
  "crossbeam",
+ "foyer-common",
+ "foyer-policy",
  "foyer-utils",
  "futures",
  "hdrhistogram",
@@ -349,6 +351,94 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "foyer-common"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "bytesize",
+ "clap",
+ "cmsketch",
+ "crossbeam",
+ "foyer-policy",
+ "foyer-utils",
+ "futures",
+ "hdrhistogram",
+ "itertools",
+ "libc",
+ "memoffset 0.8.0",
+ "nix",
+ "parking_lot",
+ "paste",
+ "prometheus",
+ "rand",
+ "rand_mt",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "twox-hash",
+]
+
+[[package]]
+name = "foyer-intrusive"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "bytesize",
+ "clap",
+ "cmsketch",
+ "crossbeam",
+ "foyer-common",
+ "foyer-utils",
+ "futures",
+ "hdrhistogram",
+ "itertools",
+ "libc",
+ "memoffset 0.8.0",
+ "nix",
+ "parking_lot",
+ "paste",
+ "prometheus",
+ "rand",
+ "rand_mt",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "twox-hash",
+]
+
+[[package]]
+name = "foyer-policy"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "bytesize",
+ "clap",
+ "cmsketch",
+ "crossbeam",
+ "foyer-utils",
+ "futures",
+ "hdrhistogram",
+ "itertools",
+ "libc",
+ "memoffset 0.8.0",
+ "nix",
+ "parking_lot",
+ "paste",
+ "prometheus",
+ "rand",
+ "rand_mt",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "twox-hash",
+]
+
 [[package]]
 name = "foyer-utils"
 version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index d4e9a2eb..55ef2284 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,13 @@
 [workspace]
 
-members = ["foyer", "foyer-bench", "foyer-utils"]
+members = [
+    "foyer",
+    "foyer-bench",
+    "foyer-common",
+    "foyer-intrusive",
+    "foyer-policy",
+    "foyer-utils",
+]
 
 [patch.crates-io]
 # cmsketch = { path = "../cmsketch-rs" }
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
new file mode 100644
index 00000000..6fc49d79
--- /dev/null
+++ b/foyer-common/Cargo.toml
@@ -0,0 +1,43 @@
+[package]
+name = "foyer-common"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1"
+bytes = "1"
+cmsketch = "0.1"
+crossbeam = "0.8"
+foyer-policy = { path = "../foyer-policy" }
+foyer-utils = { path = "../foyer-utils" }
+futures = "0.3"
+itertools = "0.10.5"
+libc = "0.2"
+memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+paste = "1.0"
+prometheus = "0.13"
+rand = "0.8.5"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+tracing = "0.1"
+twox-hash = "1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+hdrhistogram = "7"
+rand_mt = "4.2.1"
+tempfile = "3"
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
new file mode 100644
index 00000000..61f53766
--- /dev/null
+++ b/foyer-common/src/lib.rs
@@ -0,0 +1,102 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(trait_alias)]
+
+use bytes::{Buf, BufMut};
+
+pub trait Item = Sized + Send + Sync + 'static;
+
+pub trait Key:
+    Sized
+    + Send
+    + Sync
+    + 'static
+    + std::hash::Hash
+    + Eq
+    + PartialEq
+    + Ord
+    + PartialOrd
+    + std::fmt::Debug
+{
+    fn serialized_len(&self) -> usize {
+        panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
+    }
+
+    fn write(&self, _buf: &mut [u8]) {
+        panic!("Method `write` must be implemented for `Key` if storage is used.")
+    }
+
+    fn read(_buf: &[u8]) -> Self {
+        panic!("Method `read` must be implemented for `Key` if storage is used.")
+    }
+}
+
+pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
+    fn serialized_len(&self) -> usize {
+        panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
+    }
+
+    fn write(&self, _buf: &mut [u8]) {
+        panic!("Method `write` must be implemented for `Value` if storage is used.")
+    }
+
+    fn read(_buf: &[u8]) -> Self {
+        panic!("Method `read` must be implemented for `Value` if storage is used.")
+    }
+}
+
+// TODO(MrCroxx): use macro to impl for all
+
+impl Key for u64 {
+    fn serialized_len(&self) -> usize {
+        8
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        buf.put_u64(*self);
+    }
+
+    fn read(mut buf: &[u8]) -> Self {
+        buf.get_u64()
+    }
+}
+
+impl Key for u32 {
+    fn serialized_len(&self) -> usize {
+        4
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        buf.put_u32(*self);
+    }
+
+    fn read(mut buf: &[u8]) -> Self {
+        buf.get_u32()
+    }
+}
+
+impl Value for Vec<u8> {
+    fn serialized_len(&self) -> usize {
+        self.len()
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        buf.put_slice(self);
+    }
+
+    fn read(buf: &[u8]) -> Self {
+        buf.to_vec()
+    }
+}
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
new file mode 100644
index 00000000..843d8461
--- /dev/null
+++ b/foyer-intrusive/Cargo.toml
@@ -0,0 +1,42 @@
+[package]
+name = "foyer-intrusive"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1"
+bytes = "1"
+cmsketch = "0.1"
+crossbeam = "0.8"
+foyer-common = { path = "../foyer-common" }
+foyer-utils = { path = "../foyer-utils" }
+futures = "0.3"
+itertools = "0.10.5"
+libc = "0.2"
+memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+paste = "1.0"
+prometheus = "0.13"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+twox-hash = "1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+hdrhistogram = "7"
+rand = "0.8.5"
+rand_mt = "4.2.1"
+tempfile = "3"
diff --git a/foyer-intrusive/README.md b/foyer-intrusive/README.md
new file mode 100644
index 00000000..5afa3268
--- /dev/null
+++ b/foyer-intrusive/README.md
@@ -0,0 +1,8 @@
+# foyer-intrusive
+
+Instrusive index collections and ordering collections for foyer.
+
+## Acknowledgments
+
+[facebook/CacheLib](https://github.com/facebook/CacheLib)
+[Amanieu/intrusive-rs](https://github.com/Amanieu/intrusive-rs)
\ No newline at end of file
diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
new file mode 100644
index 00000000..f000e062
--- /dev/null
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -0,0 +1,513 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::ptr::NonNull;
+
+use crate::core::adapter::{Adapter, Link};
+use crate::core::pointer::PointerOps;
+
+#[derive(Debug, Default)]
+pub struct DListLink {
+    prev: Option<NonNull<DListLink>>,
+    next: Option<NonNull<DListLink>>,
+    is_linked: bool,
+}
+
+impl DListLink {
+    pub fn raw(&self) -> NonNull<DListLink> {
+        unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
+    }
+}
+
+unsafe impl Send for DListLink {}
+unsafe impl Sync for DListLink {}
+
+impl Link for DListLink {
+    fn is_linked(&self) -> bool {
+        self.is_linked
+    }
+}
+
+#[derive(Debug)]
+pub struct DList<A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    head: Option<NonNull<DListLink>>,
+    tail: Option<NonNull<DListLink>>,
+
+    len: usize,
+
+    adapter: A,
+}
+
+impl<A> DList<A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    pub fn new() -> Self {
+        Self {
+            head: None,
+            tail: None,
+            len: 0,
+
+            adapter: A::new(),
+        }
+    }
+
+    pub fn front(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+        unsafe {
+            self.head
+                .map(|link| self.adapter.link2item(link.as_ptr()))
+                .map(|link| &*link)
+        }
+    }
+
+    pub fn back(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+        unsafe {
+            self.tail
+                .map(|link| self.adapter.link2item(link.as_ptr()))
+                .map(|link| &*link)
+        }
+    }
+
+    pub fn push_front(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        self.iter_mut().insert_after(ptr);
+    }
+
+    pub fn push_back(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        self.iter_mut().insert_before(ptr);
+    }
+
+    pub fn pop_front(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+        let mut iter = self.iter_mut();
+        iter.next();
+        iter.remove()
+    }
+
+    pub fn pop_back(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+        let mut iter = self.iter_mut();
+        iter.prev();
+        iter.remove()
+    }
+
+    pub fn iter(&self) -> DListIter<'_, A> {
+        DListIter {
+            link: None,
+            dlist: self,
+        }
+    }
+
+    pub fn iter_mut(&mut self) -> DListIterMut<'_, A> {
+        DListIterMut {
+            link: None,
+            dlist: self,
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Create mutable iterator directly on raw link.
+    ///
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`DList`].
+    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DListLink>) -> DListIterMut<'_, A> {
+        DListIterMut {
+            link: Some(link),
+            dlist: self,
+        }
+    }
+
+    /// Create immutable iterator directly on raw link.
+    ///
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`DList`].
+    pub unsafe fn iter_from_raw(&self, link: NonNull<DListLink>) -> DListIter<'_, A> {
+        DListIter {
+            link: Some(link),
+            dlist: self,
+        }
+    }
+}
+
+pub struct DListIter<'a, A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    link: Option<NonNull<A::Link>>,
+    dlist: &'a DList<A>,
+}
+
+impl<'a, A> DListIter<'a, A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    pub fn is_valid(&self) -> bool {
+        self.link.is_some()
+    }
+
+    pub fn get(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+        self.link
+            .map(|link| unsafe { &*self.dlist.adapter.link2item(link.as_ptr()) })
+    }
+
+    /// Move to next.
+    ///
+    /// If iter is on tail, move to null.
+    /// If iter is on null, move to head.
+    pub fn next(&mut self) {
+        unsafe {
+            match self.link {
+                Some(link) => self.link = link.as_ref().next,
+                None => self.link = self.dlist.head,
+            }
+        }
+    }
+
+    /// Move to prev.
+    ///
+    /// If iter is on head, move to null.
+    /// If iter is on null, move to tail.
+    pub fn prev(&mut self) {
+        unsafe {
+            match self.link {
+                Some(link) => self.link = link.as_ref().prev,
+                None => self.link = self.dlist.tail,
+            }
+        }
+    }
+
+    /// Move to head.
+    pub fn head(&mut self) {
+        self.link = self.dlist.head;
+    }
+
+    /// Move to head.
+    pub fn tail(&mut self) {
+        self.link = self.dlist.tail;
+    }
+}
+
+pub struct DListIterMut<'a, A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    link: Option<NonNull<A::Link>>,
+    dlist: &'a mut DList<A>,
+}
+
+impl<'a, A> DListIterMut<'a, A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    pub fn is_valid(&self) -> bool {
+        self.link.is_some()
+    }
+
+    pub fn get(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+        self.link
+            .map(|link| unsafe { &*self.dlist.adapter.link2item(link.as_ptr()) })
+    }
+
+    pub fn get_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+        self.link
+            .map(|link| unsafe { &mut *(self.dlist.adapter.link2item(link.as_ptr()) as *mut _) })
+    }
+
+    /// Move to next.
+    ///
+    /// If iter is on tail, move to null.
+    /// If iter is on null, move to head.
+    pub fn next(&mut self) {
+        unsafe {
+            match self.link {
+                Some(link) => self.link = link.as_ref().next,
+                None => self.link = self.dlist.head,
+            }
+        }
+    }
+
+    /// Move to prev.
+    ///
+    /// If iter is on head, move to null.
+    /// If iter is on null, move to tail.
+    pub fn prev(&mut self) {
+        unsafe {
+            match self.link {
+                Some(link) => self.link = link.as_ref().prev,
+                None => self.link = self.dlist.tail,
+            }
+        }
+    }
+
+    /// Move to front.
+    pub fn front(&mut self) {
+        self.link = self.dlist.head;
+    }
+
+    /// Move to back.
+    pub fn back(&mut self) {
+        self.link = self.dlist.tail;
+    }
+
+    /// Removes the current item from [`DList`] and move next.
+    pub fn remove(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+        unsafe {
+            if !self.is_valid() {
+                return None;
+            }
+
+            let mut link = self.link.unwrap();
+
+            let item = self.dlist.adapter.link2item(link.as_ptr());
+            let ptr = self.dlist.adapter.pointer_ops().from_raw(item);
+
+            // fix head and tail if node is either of that
+            let mut prev = link.as_ref().prev;
+            let mut next = link.as_ref().next;
+            if Some(link) == self.dlist.head {
+                self.dlist.head = next;
+            }
+            if Some(link) == self.dlist.tail {
+                self.dlist.tail = prev;
+            }
+
+            // fix the next and prev ptrs of the node before and after this
+            if let Some(prev) = &mut prev {
+                prev.as_mut().next = next;
+            }
+            if let Some(next) = &mut next {
+                next.as_mut().prev = prev;
+            }
+
+            link.as_mut().next = None;
+            link.as_mut().prev = None;
+            link.as_mut().is_linked = false;
+
+            self.dlist.len -= 1;
+
+            self.link = next;
+
+            Some(ptr)
+        }
+    }
+
+    /// Link a new ptr before the current one.
+    ///
+    /// If iter is on null, link to tail.
+    pub fn insert_before(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item_new = self.dlist.adapter.pointer_ops().into_raw(ptr);
+            let mut link_new =
+                NonNull::new_unchecked(self.dlist.adapter.item2link(item_new) as *mut A::Link);
+            assert!(!link_new.as_ref().is_linked());
+
+            match self.link {
+                Some(link) => self.link_before(link_new, link),
+                None => {
+                    self.link_between(link_new, self.dlist.tail, None);
+                    self.dlist.tail = Some(link_new);
+                }
+            }
+
+            if self.dlist.head == self.link {
+                self.dlist.head = Some(link_new);
+            }
+
+            link_new.as_mut().is_linked = true;
+
+            self.dlist.len += 1;
+        }
+    }
+
+    /// Link a new ptr after the current one.
+    ///
+    /// If iter is on null, link to head.
+    pub fn insert_after(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item_new = self.dlist.adapter.pointer_ops().into_raw(ptr);
+            let mut link_new =
+                NonNull::new_unchecked(self.dlist.adapter.item2link(item_new) as *mut A::Link);
+            assert!(!link_new.as_ref().is_linked());
+
+            match self.link {
+                Some(link) => self.link_after(link_new, link),
+                None => {
+                    self.link_between(link_new, None, self.dlist.head);
+                    self.dlist.head = Some(link_new);
+                }
+            }
+
+            if self.dlist.tail == self.link {
+                self.dlist.tail = Some(link_new);
+            }
+
+            link_new.as_mut().is_linked = true;
+
+            self.dlist.len += 1;
+        }
+    }
+
+    unsafe fn link_before(&mut self, link: NonNull<A::Link>, next: NonNull<A::Link>) {
+        self.link_between(link, next.as_ref().prev, Some(next));
+    }
+
+    unsafe fn link_after(&mut self, link: NonNull<A::Link>, prev: NonNull<A::Link>) {
+        self.link_between(link, Some(prev), prev.as_ref().next);
+    }
+
+    unsafe fn link_between(
+        &mut self,
+        mut link: NonNull<A::Link>,
+        mut prev: Option<NonNull<A::Link>>,
+        mut next: Option<NonNull<A::Link>>,
+    ) {
+        if let Some(prev) = &mut prev {
+            prev.as_mut().next = Some(link);
+        }
+        if let Some(next) = &mut next {
+            next.as_mut().prev = Some(link);
+        }
+        link.as_mut().prev = prev;
+        link.as_mut().next = next;
+    }
+}
+
+impl<'a, A> Iterator for DListIter<'a, A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    type Item = &'a <A::PointerOps as PointerOps>::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next();
+        match self.link {
+            Some(link) => Some(unsafe { &*(self.dlist.adapter.link2item(link.as_ptr())) }),
+            None => None,
+        }
+    }
+}
+
+impl<'a, A> Iterator for DListIterMut<'a, A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    type Item = &'a mut <A::PointerOps as PointerOps>::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next();
+        match self.link {
+            Some(link) => {
+                Some(unsafe { &mut *(self.dlist.adapter.link2item(link.as_ptr()) as *mut _) })
+            }
+            None => None,
+        }
+    }
+}
+
+// TODO(MrCroxx): Need more tests.
+
+#[cfg(test)]
+mod tests {
+    use itertools::Itertools;
+
+    use crate::core::pointer::DefaultPointerOps;
+
+    use super::*;
+
+    #[derive(Debug)]
+    struct DListItem {
+        link: DListLink,
+        val: u64,
+    }
+
+    impl DListItem {
+        fn new(val: u64) -> Self {
+            Self {
+                link: DListLink::default(),
+                val,
+            }
+        }
+    }
+
+    #[derive(Debug, Default)]
+    struct DListAdapter(DefaultPointerOps<Box<DListItem>>);
+
+    unsafe impl Adapter for DListAdapter {
+        type PointerOps = DefaultPointerOps<Box<DListItem>>;
+
+        type Link = DListLink;
+
+        fn new() -> Self {
+            Self::default()
+        }
+
+        fn pointer_ops(&self) -> &Self::PointerOps {
+            &self.0
+        }
+
+        unsafe fn link2item(
+            &self,
+            link: *const Self::Link,
+        ) -> *const <Self::PointerOps as PointerOps>::Item {
+            crate::container_of!(link, DListItem, link)
+        }
+
+        unsafe fn item2link(
+            &self,
+            item: *const <Self::PointerOps as PointerOps>::Item,
+        ) -> *const Self::Link {
+            (item as *const u8).add(crate::offset_of!(DListItem, link)) as *const _
+        }
+    }
+
+    #[test]
+    fn test_dlist_simple() {
+        let mut l = DList::<DListAdapter>::new();
+
+        l.push_back(Box::new(DListItem::new(2)));
+        l.push_front(Box::new(DListItem::new(1)));
+        l.push_back(Box::new(DListItem::new(3)));
+
+        let v = l.iter_mut().map(|item| item.val).collect_vec();
+        assert_eq!(v, vec![1, 2, 3]);
+        assert_eq!(l.len(), 3);
+
+        let mut iter = l.iter_mut();
+        iter.next();
+        iter.next();
+        assert_eq!(iter.get().unwrap().val, 2);
+        let i2 = iter.remove();
+        assert_eq!(i2.unwrap().val, 2);
+        assert_eq!(iter.get().unwrap().val, 3);
+        let v = l.iter_mut().map(|item| item.val).collect_vec();
+        assert_eq!(v, vec![1, 3]);
+        assert_eq!(l.len(), 2);
+
+        let i3 = l.pop_back();
+        assert_eq!(i3.unwrap().val, 3);
+        let i1 = l.pop_front();
+        assert_eq!(i1.unwrap().val, 1);
+        assert!(l.pop_front().is_none());
+        assert_eq!(l.len(), 0);
+    }
+}
diff --git a/foyer-intrusive/src/collections/mod.rs b/foyer-intrusive/src/collections/mod.rs
new file mode 100644
index 00000000..4fe9e427
--- /dev/null
+++ b/foyer-intrusive/src/collections/mod.rs
@@ -0,0 +1,15 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+pub mod dlist;
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
new file mode 100644
index 00000000..b759a8cd
--- /dev/null
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -0,0 +1,269 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use foyer_common::Key;
+
+use crate::core::pointer::PointerOps;
+
+pub trait Link: Send + Sync + 'static {
+    fn is_linked(&self) -> bool;
+}
+
+/// # Safety
+///
+/// Pointer operations MUST be valid.
+///
+/// [`Adapter`] is recommanded to be generated by macro `instrusive_adapter!`.
+pub unsafe trait Adapter: Send + Sync + 'static {
+    type PointerOps: PointerOps;
+    type Link: Link;
+
+    fn new() -> Self;
+
+    fn pointer_ops(&self) -> &Self::PointerOps;
+
+    /// # Safety
+    ///
+    /// Pointer operations MUST be valid.
+    unsafe fn link2item(
+        &self,
+        link: *const Self::Link,
+    ) -> *const <Self::PointerOps as PointerOps>::Item;
+
+    /// # Safety
+    ///
+    /// Pointer operations MUST be valid.
+    unsafe fn item2link(
+        &self,
+        item: *const <Self::PointerOps as PointerOps>::Item,
+    ) -> *const Self::Link;
+}
+
+/// # Safety
+///
+/// Pointer operations MUST be valid.
+///
+/// [`KeyAdapter`] is recommanded to be generated by macro `key_adapter!`.
+pub unsafe trait KeyAdapter: Adapter {
+    type Key: Key;
+
+    /// # Safety
+    ///
+    /// Pointer operations MUST be valid.
+    unsafe fn item2key(
+        &self,
+        item: *const <Self::PointerOps as PointerOps>::Item,
+    ) -> *const Self::Key;
+}
+
+/// Macro to generate an implementation of [`Adapter`] for instrusive container and items.
+///
+/// The basic syntax to create an adapter is:
+///
+/// ```rust,ignore
+/// intrusive_adapter! { Adapter = Pointer: Item { link_field: LinkType } }
+/// ```
+///
+/// # Generics
+///
+/// This macro supports generic arguments:
+///
+/// Note that due to macro parsing limitations, `T: Trait` bounds are not
+/// supported in the generic argument list. You must list any trait bounds in
+/// a separate `where` clause at the end of the macro.
+///
+/// # Examples
+///
+/// ```
+/// use foyer_intrusive::{intrusive_adapter, key_adapter};
+/// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter};
+/// use foyer_intrusive::core::pointer::PointerOps;
+/// use foyer_intrusive::eviction::EvictionPolicy;
+/// use std::sync::Arc;
+///
+/// pub struct Item<A, E>
+/// where
+///     E: EvictionPolicy<A>,
+///     A: KeyAdapter<Link = E::Link>,
+///     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+/// {
+///     link: E::Link,
+///     key: u64,
+/// }
+///
+/// intrusive_adapter! { ItemAdapter<E> = Arc<Item<ItemAdapter<E>, E>>: Item<ItemAdapter<E>, E> { link: E::Link} where E:EvictionPolicy<ItemAdapter<E>> }
+/// key_adapter! { ItemAdapter<E> = Item<ItemAdapter<E>, E> { key: u64 } where E: EvictionPolicy<ItemAdapter<E>> }
+/// ```
+#[macro_export]
+macro_rules! intrusive_adapter {
+    (@impl
+        $vis:vis $name:ident ($($args:tt),*) = $pointer:ty: $item:path { $field:ident: $link:ty } $($where_:tt)*
+    ) => {
+        $vis struct $name<$($args),*> $($where_)* {
+            pointer_ops: $crate::core::pointer::DefaultPointerOps<$pointer>,
+        }
+
+        unsafe impl<$($args),*> Send for $name<$($args),*> $($where_)* {}
+        unsafe impl<$($args),*> Sync for $name<$($args),*> $($where_)* {}
+
+        unsafe impl<$($args),*> $crate::core::adapter::Adapter for $name<$($args),*> $($where_)*{
+            type PointerOps = $crate::core::pointer::DefaultPointerOps<$pointer>;
+            type Link = $link;
+
+            fn new() -> Self {
+                Self {
+                    pointer_ops: Default::default(),
+                }
+            }
+
+            fn pointer_ops(&self) -> &Self::PointerOps {
+                &self.pointer_ops
+            }
+
+            unsafe fn link2item(
+                &self,
+                link: *const Self::Link,
+            ) -> *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item {
+                $crate::container_of!(link, $item, $field)
+            }
+
+            unsafe fn item2link(
+                &self,
+                item: *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item,
+            ) -> *const Self::Link {
+                (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
+            }
+        }
+    };
+    (
+        $vis:vis $name:ident = $($rest:tt)*
+    ) => {
+        intrusive_adapter! {@impl
+            $vis $name () = $($rest)*
+        }
+    };
+    (
+        $vis:vis $name:ident<$($args:tt),*> = $($rest:tt)*
+    ) => {
+        intrusive_adapter! {@impl
+            $vis $name ($($args)*) = $($rest)*
+        }
+    };
+}
+
+/// Macro to generate an implementation of [`KeyAdapter`] for instrusive container and items.
+/// ///
+/// The basic syntax to create an adapter is:
+///
+/// ```rust,ignore
+/// key_adapter! { Adapter = Item { key_field: KeyType } }
+/// ```
+///
+/// # Generics
+///
+/// This macro supports generic arguments:
+///
+/// Note that due to macro parsing limitations, `T: Trait` bounds are not
+/// supported in the generic argument list. You must list any trait bounds in
+/// a separate `where` clause at the end of the macro.
+///
+/// # Examples
+///
+/// ```
+/// use foyer_intrusive::{intrusive_adapter, key_adapter};
+/// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter};
+/// use foyer_intrusive::core::pointer::PointerOps;
+/// use foyer_intrusive::eviction::EvictionPolicy;
+/// use std::sync::Arc;
+///
+/// pub struct Item<A, E>
+/// where
+///     E: EvictionPolicy<A>,
+///     A: KeyAdapter<Link = E::Link>,
+///     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+/// {
+///     link: E::Link,
+///     key: u64,
+/// }
+///
+/// intrusive_adapter! { ItemAdapter<E> = Arc<Item<ItemAdapter<E>, E>>: Item<ItemAdapter<E>, E> { link: E::Link} where E:EvictionPolicy<ItemAdapter<E>> }
+/// key_adapter! { ItemAdapter<E> = Item<ItemAdapter<E>, E> { key: u64 } where E: EvictionPolicy<ItemAdapter<E>> }
+/// ```
+#[macro_export]
+macro_rules! key_adapter {
+    (@impl
+        $adapter:ident ($($args:tt),*) = $item:ty { $field:ident: $key:ty } $($where_:tt)*
+    ) => {
+        unsafe impl<$($args),*> $crate::core::adapter::KeyAdapter for $adapter<$($args),*> $($where_)*{
+            type Key = $key;
+
+            unsafe fn item2key(
+                &self,
+                item: *const <Self::PointerOps as PointerOps>::Item,
+            ) -> *const Self::Key {
+                (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
+            }
+        }
+    };
+    (
+        $name:ident = $($rest:tt)*
+    ) => {
+        key_adapter! {@impl
+            $name () = $($rest)*
+        }
+    };
+    (
+        $name:ident<$($args:tt),*> = $($rest:tt)*
+    ) => {
+        key_adapter! {@impl
+            $name ($($args)*) = $($rest)*
+        }
+    };
+}
+
+#[cfg(test)]
+mod tests {
+    use itertools::Itertools;
+
+    use crate::collections::dlist::*;
+
+    use crate::intrusive_adapter;
+
+    use super::*;
+
+    struct DListItem {
+        link: DListLink,
+        val: u64,
+    }
+
+    impl DListItem {
+        fn new(val: u64) -> Self {
+            Self {
+                link: DListLink::default(),
+                val,
+            }
+        }
+    }
+
+    intrusive_adapter! { DListItemAdapter = Box<DListItem>: DListItem { link: DListLink }}
+    key_adapter! { DListItemAdapter = DListItem { val: u64 } }
+
+    #[test]
+    fn test_adapter_macro() {
+        let mut l = DList::<DListItemAdapter>::new();
+        l.push_front(Box::new(DListItem::new(1)));
+        let v = l.iter().map(|item| item.val).collect_vec();
+        assert_eq!(v, vec![1]);
+    }
+}
diff --git a/foyer-intrusive/src/core/mod.rs b/foyer-intrusive/src/core/mod.rs
new file mode 100644
index 00000000..6c823b94
--- /dev/null
+++ b/foyer-intrusive/src/core/mod.rs
@@ -0,0 +1,16 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+pub mod adapter;
+pub mod pointer;
diff --git a/foyer-intrusive/src/core/pointer.rs b/foyer-intrusive/src/core/pointer.rs
new file mode 100644
index 00000000..027644e4
--- /dev/null
+++ b/foyer-intrusive/src/core/pointer.rs
@@ -0,0 +1,565 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright 2020 Amari Robinson
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::marker::PhantomData;
+use std::pin::Pin;
+use std::ptr::NonNull;
+use std::rc::Rc;
+use std::sync::Arc;
+
+/// # Safety
+///
+/// Pointer operations MUST be valid.
+pub unsafe trait PointerOps {
+    type Item: ?Sized;
+    type Pointer;
+
+    /// # Safety
+    ///
+    /// Pointer operations MUST be valid.
+    unsafe fn from_raw(&self, item: *const Self::Item) -> Self::Pointer;
+
+    fn into_raw(&self, ptr: Self::Pointer) -> *const Self::Item;
+
+    fn as_ptr(&self, ptr: &Self::Pointer) -> *const Self::Item;
+}
+
+/// # Safety
+///
+/// Pointer operations MUST be valid.
+pub unsafe trait DowngradablePointerOps: PointerOps {
+    type WeakPointer;
+
+    fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer;
+}
+
+#[derive(Debug)]
+pub struct DefaultPointerOps<Pointer>(PhantomData<Pointer>);
+
+impl<Pointer> DefaultPointerOps<Pointer> {
+    /// Constructs an instance of `DefaultPointerOps`.
+    #[inline]
+    pub const fn new() -> DefaultPointerOps<Pointer> {
+        DefaultPointerOps(PhantomData)
+    }
+}
+
+impl<Pointer> Clone for DefaultPointerOps<Pointer> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<Pointer> Copy for DefaultPointerOps<Pointer> {}
+
+impl<Pointer> Default for DefaultPointerOps<Pointer> {
+    #[inline]
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+unsafe impl<'a, T: ?Sized> PointerOps for DefaultPointerOps<&'a T> {
+    type Item = T;
+    type Pointer = &'a T;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> &'a T {
+        &*raw
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: &'a T) -> *const T {
+        ptr
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &&'a T) -> *const T {
+        *ptr as *const _
+    }
+}
+
+unsafe impl<'a, T: ?Sized> PointerOps for DefaultPointerOps<Pin<&'a T>> {
+    type Item = T;
+    type Pointer = Pin<&'a T>;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> Pin<&'a T> {
+        Pin::new_unchecked(&*raw)
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: Pin<&'a T>) -> *const T {
+        unsafe { Pin::into_inner_unchecked(ptr) as *const T }
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &Pin<&'a T>) -> *const T {
+        ptr.get_ref() as *const _
+    }
+}
+
+unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<NonNull<T>> {
+    type Item = T;
+    type Pointer = NonNull<T>;
+
+    unsafe fn from_raw(&self, raw: *const T) -> NonNull<T> {
+        NonNull::new_unchecked(raw as *mut _)
+    }
+
+    fn into_raw(&self, ptr: NonNull<T>) -> *const T {
+        ptr.as_ptr()
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &NonNull<T>) -> *const T {
+        ptr.as_ptr()
+    }
+}
+
+unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Box<T>> {
+    type Item = T;
+    type Pointer = Box<T>;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> Box<T> {
+        Box::from_raw(raw as *mut T)
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: Box<T>) -> *const T {
+        Box::into_raw(ptr) as *const T
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &Box<T>) -> *const T {
+        ptr.as_ref() as *const _
+    }
+}
+
+unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Box<T>>> {
+    type Item = T;
+    type Pointer = Pin<Box<T>>;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> Pin<Box<T>> {
+        Pin::new_unchecked(Box::from_raw(raw as *mut T))
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: Pin<Box<T>>) -> *const T {
+        Box::into_raw(unsafe { Pin::into_inner_unchecked(ptr) }) as *const T
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &Pin<Box<T>>) -> *const T {
+        ptr.as_ref().get_ref() as *const _
+    }
+}
+
+unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Rc<T>> {
+    type Item = T;
+    type Pointer = Rc<T>;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> Rc<T> {
+        Rc::from_raw(raw)
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: Rc<T>) -> *const T {
+        Rc::into_raw(ptr)
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &Rc<T>) -> *const T {
+        Rc::as_ptr(ptr)
+    }
+}
+
+unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Rc<T>>> {
+    type Item = T;
+    type Pointer = Pin<Rc<T>>;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> Pin<Rc<T>> {
+        Pin::new_unchecked(Rc::from_raw(raw))
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: Pin<Rc<T>>) -> *const T {
+        Rc::into_raw(unsafe { Pin::into_inner_unchecked(ptr) })
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &Pin<Rc<T>>) -> *const T {
+        ptr.as_ref().get_ref() as *const _
+    }
+}
+
+unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Arc<T>> {
+    type Item = T;
+    type Pointer = Arc<T>;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> Arc<T> {
+        Arc::from_raw(raw)
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: Arc<T>) -> *const T {
+        Arc::into_raw(ptr)
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &Arc<T>) -> *const T {
+        Arc::as_ptr(ptr)
+    }
+}
+
+unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Arc<T>>> {
+    type Item = T;
+    type Pointer = Pin<Arc<T>>;
+
+    #[inline]
+    unsafe fn from_raw(&self, raw: *const T) -> Pin<Arc<T>> {
+        Pin::new_unchecked(Arc::from_raw(raw))
+    }
+
+    #[inline]
+    fn into_raw(&self, ptr: Pin<Arc<T>>) -> *const T {
+        Arc::into_raw(unsafe { Pin::into_inner_unchecked(ptr) })
+    }
+
+    #[inline]
+    fn as_ptr(&self, ptr: &Pin<Arc<T>>) -> *const T {
+        ptr.as_ref().get_ref() as *const _
+    }
+}
+
+unsafe impl<T: ?Sized> DowngradablePointerOps for DefaultPointerOps<Rc<T>> {
+    type WeakPointer = std::rc::Weak<T>;
+
+    fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer {
+        Rc::downgrade(ptr)
+    }
+}
+
+unsafe impl<T: ?Sized> DowngradablePointerOps for DefaultPointerOps<Arc<T>> {
+    type WeakPointer = std::sync::Weak<T>;
+
+    fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer {
+        Arc::downgrade(ptr)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::boxed::Box;
+    use std::fmt::Debug;
+    use std::mem;
+    use std::pin::Pin;
+    use std::rc::Rc;
+    use std::sync::Arc;
+
+    /// Clones a `PointerOps::Pointer` from a `*const PointerOps::Value`
+    ///
+    /// This method is only safe to call if the raw pointer is known to be
+    /// managed by the provided `PointerOps` type.
+    #[inline]
+    unsafe fn clone_pointer_from_raw<T: PointerOps>(
+        pointer_ops: &T,
+        ptr: *const T::Item,
+    ) -> T::Pointer
+    where
+        T::Pointer: Clone,
+    {
+        use std::mem::ManuallyDrop;
+        use std::ops::Deref;
+
+        /// Guard which converts an pointer back into its raw version
+        /// when it gets dropped. This makes sure we also perform a full
+        /// `from_raw` and `into_raw` round trip - even in the case of panics.
+        struct PointerGuard<'a, T: PointerOps> {
+            pointer: ManuallyDrop<T::Pointer>,
+            pointer_ops: &'a T,
+        }
+
+        impl<'a, T: PointerOps> Drop for PointerGuard<'a, T> {
+            #[inline]
+            fn drop(&mut self) {
+                // Prevent shared pointers from being released by converting them
+                // back into the raw pointers
+                // SAFETY: `pointer` is never dropped. `ManuallyDrop::take` is not stable until 1.42.0.
+                let _ = self
+                    .pointer_ops
+                    .into_raw(unsafe { core::ptr::read(&*self.pointer) });
+            }
+        }
+
+        let holder = PointerGuard {
+            pointer: ManuallyDrop::new(pointer_ops.from_raw(ptr)),
+            pointer_ops,
+        };
+        holder.pointer.deref().clone()
+    }
+
+    #[test]
+    fn test_box() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Box<_>>::new();
+            let p = Box::new(1);
+            let a: *const i32 = &*p;
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            let p2: Box<i32> = pointer_ops.from_raw(r);
+            let a2: *const i32 = &*p2;
+            assert_eq!(a, a2);
+        }
+    }
+
+    #[test]
+    fn test_rc() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Rc<_>>::new();
+            let p = Rc::new(1);
+            let a: *const i32 = &*p;
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            let p2: Rc<i32> = pointer_ops.from_raw(r);
+            let a2: *const i32 = &*p2;
+            assert_eq!(a, a2);
+        }
+    }
+
+    #[test]
+    fn test_arc() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Arc<_>>::new();
+            let p = Arc::new(1);
+            let a: *const i32 = &*p;
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            let p2: Arc<i32> = pointer_ops.from_raw(r);
+            let a2: *const i32 = &*p2;
+            assert_eq!(a, a2);
+        }
+    }
+
+    #[test]
+    fn test_box_unsized() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Box<_>>::new();
+            let p = Box::new(1) as Box<dyn Debug>;
+            let a: *const dyn Debug = &*p;
+            let b: (usize, usize) = mem::transmute(a);
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            assert_eq!(b, mem::transmute(r));
+            let p2: Box<dyn Debug> = pointer_ops.from_raw(r);
+            let a2: *const dyn Debug = &*p2;
+            assert_eq!(a, a2);
+            assert_eq!(b, mem::transmute(a2));
+        }
+    }
+
+    #[test]
+    fn test_rc_unsized() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Rc<_>>::new();
+            let p = Rc::new(1) as Rc<dyn Debug>;
+            let a: *const dyn Debug = &*p;
+            let b: (usize, usize) = mem::transmute(a);
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            assert_eq!(b, mem::transmute(r));
+            let p2: Rc<dyn Debug> = pointer_ops.from_raw(r);
+            let a2: *const dyn Debug = &*p2;
+            assert_eq!(a, a2);
+            assert_eq!(b, mem::transmute(a2));
+        }
+    }
+
+    #[test]
+    fn test_arc_unsized() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Arc<_>>::new();
+            let p = Arc::new(1) as Arc<dyn Debug>;
+            let a: *const dyn Debug = &*p;
+            let b: (usize, usize) = mem::transmute(a);
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            assert_eq!(b, mem::transmute(r));
+            let p2: Arc<dyn Debug> = pointer_ops.from_raw(r);
+            let a2: *const dyn Debug = &*p2;
+            assert_eq!(a, a2);
+            assert_eq!(b, mem::transmute(a2));
+        }
+    }
+
+    #[test]
+    fn clone_arc_from_raw() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Arc<_>>::new();
+            let p = Arc::new(1);
+            let raw = Arc::as_ptr(&p);
+            let p2: Arc<i32> = clone_pointer_from_raw(&pointer_ops, raw);
+            assert_eq!(2, Arc::strong_count(&p2));
+        }
+    }
+
+    #[test]
+    fn clone_rc_from_raw() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Rc<_>>::new();
+            let p = Rc::new(1);
+            let raw = Rc::as_ptr(&p);
+            let p2: Rc<i32> = clone_pointer_from_raw(&pointer_ops, raw);
+            assert_eq!(2, Rc::strong_count(&p2));
+        }
+    }
+
+    #[test]
+    fn test_pin_box() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Box<_>>>::new();
+            let p = Pin::new(Box::new(1));
+            let a: *const i32 = &*p;
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            let p2: Pin<Box<i32>> = pointer_ops.from_raw(r);
+            let a2: *const i32 = &*p2;
+            assert_eq!(a, a2);
+        }
+    }
+
+    #[test]
+    fn test_pin_rc() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Rc<_>>>::new();
+            let p = Pin::new(Rc::new(1));
+            let a: *const i32 = &*p;
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            let p2: Pin<Rc<i32>> = pointer_ops.from_raw(r);
+            let a2: *const i32 = &*p2;
+            assert_eq!(a, a2);
+        }
+    }
+
+    #[test]
+    fn test_pin_arc() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Arc<_>>>::new();
+            let p = Pin::new(Arc::new(1));
+            let a: *const i32 = &*p;
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            let p2: Pin<Arc<i32>> = pointer_ops.from_raw(r);
+            let a2: *const i32 = &*p2;
+            assert_eq!(a, a2);
+        }
+    }
+
+    #[test]
+    fn test_pin_box_unsized() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Box<_>>>::new();
+            let p = Pin::new(Box::new(1)) as Pin<Box<dyn Debug>>;
+            let a: *const dyn Debug = &*p;
+            let b: (usize, usize) = mem::transmute(a);
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            assert_eq!(b, mem::transmute(r));
+            let p2: Pin<Box<dyn Debug>> = pointer_ops.from_raw(r);
+            let a2: *const dyn Debug = &*p2;
+            assert_eq!(a, a2);
+            assert_eq!(b, mem::transmute(a2));
+        }
+    }
+
+    #[test]
+    fn test_pin_rc_unsized() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Rc<_>>>::new();
+            let p = Pin::new(Rc::new(1)) as Pin<Rc<dyn Debug>>;
+            let a: *const dyn Debug = &*p;
+            let b: (usize, usize) = mem::transmute(a);
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            assert_eq!(b, mem::transmute(r));
+            let p2: Pin<Rc<dyn Debug>> = pointer_ops.from_raw(r);
+            let a2: *const dyn Debug = &*p2;
+            assert_eq!(a, a2);
+            assert_eq!(b, mem::transmute(a2));
+        }
+    }
+
+    #[test]
+    fn test_pin_arc_unsized() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Arc<_>>>::new();
+            let p = Pin::new(Arc::new(1)) as Pin<Arc<dyn Debug>>;
+            let a: *const dyn Debug = &*p;
+            let b: (usize, usize) = mem::transmute(a);
+            let r = pointer_ops.into_raw(p);
+            assert_eq!(a, r);
+            assert_eq!(b, mem::transmute(r));
+            let p2: Pin<Arc<dyn Debug>> = pointer_ops.from_raw(r);
+            let a2: *const dyn Debug = &*p2;
+            assert_eq!(a, a2);
+            assert_eq!(b, mem::transmute(a2));
+        }
+    }
+
+    #[test]
+    fn clone_pin_arc_from_raw() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Arc<_>>>::new();
+            let p = Pin::new(Arc::new(1));
+            let raw = pointer_ops.into_raw(p);
+            let p2: Pin<Arc<i32>> = clone_pointer_from_raw(&pointer_ops, raw);
+            let _p = pointer_ops.from_raw(raw);
+            assert_eq!(2, Arc::strong_count(&Pin::into_inner(p2)));
+        }
+    }
+
+    #[test]
+    fn clone_pin_rc_from_raw() {
+        unsafe {
+            let pointer_ops = DefaultPointerOps::<Pin<Rc<_>>>::new();
+            let p = Pin::new(Rc::new(1));
+            let raw = pointer_ops.into_raw(p);
+            let p2: Pin<Rc<i32>> = clone_pointer_from_raw(&pointer_ops, raw);
+            let _p = pointer_ops.from_raw(raw);
+            assert_eq!(2, Rc::strong_count(&Pin::into_inner(p2)));
+        }
+    }
+}
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
new file mode 100644
index 00000000..324d650e
--- /dev/null
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -0,0 +1,618 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use crate::collections::dlist::DList;
+use crate::collections::dlist::DListIter;
+use crate::core::adapter::Adapter;
+use crate::core::adapter::KeyAdapter;
+use crate::core::adapter::Link;
+use crate::core::pointer::PointerOps;
+use crate::intrusive_adapter;
+
+use std::mem::ManuallyDrop;
+use std::ptr::NonNull;
+
+use cmsketch::CMSketchUsize;
+use std::hash::Hash;
+use std::hash::Hasher;
+use twox_hash::XxHash64;
+
+use crate::collections::dlist::DListLink;
+
+use super::EvictionPolicy;
+
+const MIN_CAPACITY: usize = 100;
+const ERROR_THRESHOLD: f64 = 5.0;
+const HASH_COUNT: usize = 4;
+const DECAY_FACTOR: f64 = 0.5;
+
+#[derive(Clone, Debug)]
+pub struct LfuConfig {
+    /// The multiplier for window len given the cache size.
+    pub window_to_cache_size_ratio: usize,
+
+    /// The ratio of tiny lru capacity to overall capacity.
+    pub tiny_lru_capacity_ratio: f64,
+}
+
+#[derive(PartialEq, Eq, Debug)]
+enum LruType {
+    Tiny,
+    Main,
+
+    None,
+}
+
+#[derive(Debug, Default)]
+pub struct LfuLink {
+    link_tiny: DListLink,
+    link_main: DListLink,
+}
+
+impl Link for LfuLink {
+    fn is_linked(&self) -> bool {
+        self.link_tiny.is_linked() || self.link_main.is_linked()
+    }
+}
+
+impl LfuLink {
+    fn lru_type(&self) -> LruType {
+        match (self.link_tiny.is_linked(), self.link_main.is_linked()) {
+            (true, true) => unreachable!(),
+            (true, false) => LruType::Tiny,
+            (false, true) => LruType::Main,
+            (false, false) => LruType::None,
+        }
+    }
+
+    fn raw(&self) -> NonNull<Self> {
+        unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
+    }
+}
+
+intrusive_adapter! { LfuLinkTinyDListAdapter = NonNull<LfuLink>: LfuLink { link_tiny: DListLink } }
+intrusive_adapter! { LfuLinkMainDListAdapter = NonNull<LfuLink>: LfuLink { link_main: DListLink } }
+
+/// Implements the W-TinyLFU cache eviction policy as described in -
+///
+/// https://arxiv.org/pdf/1512.00727.pdf
+///
+/// The cache is split into 2 parts, the main cache and the tiny cache.
+/// The tiny cache is typically sized to be 1% of the total cache with
+/// the main cache being the rest 99%. Both caches are implemented using
+/// LRUs. New items land in tiny cache. During eviction, the tail item
+/// from the tiny cache is promoted to main cache if its frequency is
+/// higher than the tail item of of main cache, and the tail of main
+/// cache is evicted. This gives the frequency based admission into main
+/// cache. Hits in each cache simply move the item to the head of each
+/// LRU cache.
+/// The frequency counts are maintained in count-min-sketch approximate
+/// counters -
+///
+/// Counter Overhead:
+/// The window_to_cache_size_ratio determines the size of counters.
+/// The default value is 32 which means the counting window size is
+/// 32 times the cache size. After every 32 X cache capacity number
+/// of items, the counts are halved to weigh frequency by recency.
+/// The function counter_size() returns the size of the counters
+/// in bytes. See maybe_grow_access_counters() implementation for
+/// how the size is computed.
+///
+/// Tiny cache size:
+/// This default to 1%. There's no need to tune this parameter.
+pub struct Lfu<A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    /// tiny lru list
+    lru_tiny: DList<LfuLinkTinyDListAdapter>,
+
+    /// main lru list
+    lru_main: DList<LfuLinkMainDListAdapter>,
+
+    /// the window length counter
+    window_size: usize,
+
+    /// maxumum value of window length which when hit the counters are halved
+    max_window_size: usize,
+
+    /// the capacity for which the counters are sized
+    capacity: usize,
+
+    /// approximate streaming frequency counters
+    ///
+    /// the counts are halved every time the max_window_len is hit
+    frequencies: CMSketchUsize,
+
+    config: LfuConfig,
+
+    adapter: A,
+}
+
+impl<A> Lfu<A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    pub fn new(config: LfuConfig) -> Self {
+        let mut res = Self {
+            lru_tiny: DList::new(),
+            lru_main: DList::new(),
+
+            window_size: 0,
+            max_window_size: 0,
+            capacity: 0,
+
+            // A dummy size, will be updated later.
+            frequencies: CMSketchUsize::new_with_size(1, 1),
+
+            config,
+
+            adapter: A::new(),
+        };
+        res.maybe_grow_access_counters();
+        res
+    }
+
+    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
+
+            assert!(!link.as_ref().is_linked());
+
+            self.lru_tiny.push_front(link);
+
+            // Initialize the frequency count for this link.
+            self.update_frequencies(link);
+
+            // If tiny cache is full, unconditionally promote tail to main cache.
+            let expected_tiny_len = (self.config.tiny_lru_capacity_ratio
+                * (self.lru_tiny.len() + self.lru_main.len()) as f64)
+                as usize;
+            if self.lru_tiny.len() > expected_tiny_len {
+                let raw = self.lru_tiny.back().unwrap().raw();
+                self.switch_to_lru_front(raw);
+            } else {
+                self.maybe_promote_tail();
+            }
+
+            // If the number of counters are too small for the cache size, double them.
+            self.maybe_grow_access_counters();
+        }
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<A::PointerOps as PointerOps>::Pointer,
+    ) -> <A::PointerOps as PointerOps>::Pointer {
+        unsafe {
+            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
+
+            assert!(link.as_ref().is_linked());
+
+            self.remove_from_lru(link);
+
+            self.adapter.pointer_ops().from_raw(item)
+        }
+    }
+
+    fn access(&mut self, ptr: &<A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
+
+            assert!(link.as_ref().is_linked());
+
+            self.move_to_lru_front(link);
+
+            self.update_frequencies(link);
+        }
+    }
+
+    fn iter(&self) -> LfuIter<A> {
+        let mut iter_main = self.lru_main.iter();
+        let mut iter_tiny = self.lru_tiny.iter();
+
+        iter_main.tail();
+        iter_tiny.tail();
+
+        LfuIter {
+            lfu: self,
+            iter_main,
+            iter_tiny,
+
+            ptr: ManuallyDrop::new(None),
+        }
+    }
+
+    fn maybe_grow_access_counters(&mut self) {
+        let capacity = self.lru_tiny.len() + self.lru_main.len();
+
+        // If the new capacity ask is more than double the current size,
+        // recreate the approximate frequency counters.
+        if 2 * self.capacity > capacity {
+            return;
+        }
+
+        self.capacity = std::cmp::max(capacity, MIN_CAPACITY);
+
+        // The window counter that's incremented on every fetch.
+        self.window_size = 0;
+
+        // The frequency counters are halved every `max_window_size` fetches to decay the frequency counts.
+        self.max_window_size = self.capacity * self.config.window_to_cache_size_ratio;
+
+        // Number of frequency counters - roughly equal to the window size divided by error tolerance.
+        let num_counters = (1f64.exp() * self.max_window_size as f64 / ERROR_THRESHOLD) as usize;
+        let num_counters = num_counters.next_power_of_two();
+
+        self.frequencies = CMSketchUsize::new_with_size(num_counters, HASH_COUNT);
+    }
+
+    unsafe fn update_frequencies(&mut self, link: NonNull<LfuLink>) {
+        self.frequencies.record(self.hash_link(link));
+        self.window_size += 1;
+
+        // Decay counts every `max_window_size`. This avoids having items that were
+        // accessed frequently (were hot) but aren't being accessed anymore (are cold)
+        // from staying in cache forever.
+        if self.window_size == self.max_window_size {
+            self.window_size >>= 1;
+            self.frequencies.decay(DECAY_FACTOR);
+        }
+    }
+
+    fn maybe_promote_tail(&mut self) {
+        unsafe {
+            let link_main = match self.lru_main.back() {
+                Some(link) => link.raw(),
+                None => return,
+            };
+            let link_tiny = match self.lru_tiny.back() {
+                Some(link) => link.raw(),
+                None => return,
+            };
+
+            if self.admit_to_main(link_main, link_tiny) {
+                self.switch_to_lru_front(link_main);
+                self.switch_to_lru_front(link_tiny);
+                return;
+            }
+
+            // A node with high frequency at the tail of main cache might prevent
+            // promotions from tiny cache from happening for a long time. Relocate
+            // the tail of main cache to prevent this.
+            self.move_to_lru_front(link_main);
+        }
+    }
+
+    fn admit_to_main(&self, link_main: NonNull<LfuLink>, link_tiny: NonNull<LfuLink>) -> bool {
+        unsafe {
+            assert_eq!(link_main.as_ref().lru_type(), LruType::Main);
+            assert_eq!(link_tiny.as_ref().lru_type(), LruType::Tiny);
+
+            let frequent_main = self.frequencies.count(self.hash_link(link_main));
+            let frequent_tiny = self.frequencies.count(self.hash_link(link_tiny));
+
+            frequent_main <= frequent_tiny
+        }
+    }
+
+    unsafe fn move_to_lru_front(&mut self, link: NonNull<LfuLink>) {
+        match link.as_ref().lru_type() {
+            LruType::Tiny => {
+                let raw = link.as_ref().link_tiny.raw();
+                let ptr = self.lru_tiny.iter_mut_from_raw(raw).remove().unwrap();
+                self.lru_tiny.push_front(ptr);
+            }
+            LruType::Main => {
+                let raw = link.as_ref().link_main.raw();
+                let ptr = self.lru_main.iter_mut_from_raw(raw).remove().unwrap();
+                self.lru_main.push_front(ptr);
+            }
+            LruType::None => unreachable!(),
+        }
+    }
+
+    unsafe fn switch_to_lru_front(&mut self, link: NonNull<LfuLink>) {
+        match link.as_ref().lru_type() {
+            LruType::Tiny => {
+                let raw = link.as_ref().link_tiny.raw();
+                let ptr = self.lru_tiny.iter_mut_from_raw(raw).remove().unwrap();
+                self.lru_main.push_front(ptr);
+            }
+            LruType::Main => {
+                let raw = link.as_ref().link_main.raw();
+                let ptr = self.lru_main.iter_mut_from_raw(raw).remove().unwrap();
+                self.lru_tiny.push_front(ptr);
+            }
+            LruType::None => unreachable!(),
+        }
+    }
+
+    unsafe fn remove_from_lru(&mut self, link: NonNull<LfuLink>) {
+        match link.as_ref().lru_type() {
+            LruType::Tiny => {
+                let raw = link.as_ref().link_tiny.raw();
+                self.lru_tiny.iter_mut_from_raw(raw).remove().unwrap();
+            }
+            LruType::Main => {
+                let raw = link.as_ref().link_main.raw();
+                self.lru_main.iter_mut_from_raw(raw).remove().unwrap();
+            }
+            LruType::None => unreachable!(),
+        }
+    }
+
+    fn hash_link(&self, link: NonNull<LfuLink>) -> u64 {
+        let mut hasher = XxHash64::default();
+        let key = unsafe {
+            let item = self.adapter.link2item(link.as_ptr());
+            let key = self.adapter.item2key(item);
+            &*key
+        };
+        key.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+pub struct LfuIter<'a, A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    lfu: &'a Lfu<A>,
+    iter_tiny: DListIter<'a, LfuLinkTinyDListAdapter>,
+    iter_main: DListIter<'a, LfuLinkMainDListAdapter>,
+
+    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+}
+
+impl<'a, A> LfuIter<'a, A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    unsafe fn update_ptr(&mut self, link: NonNull<LfuLink>) {
+        std::mem::forget(self.ptr.take());
+
+        let item = self.lfu.adapter.link2item(link.as_ptr());
+        let ptr = self.lfu.adapter.pointer_ops().from_raw(item);
+        self.ptr = ManuallyDrop::new(Some(ptr));
+    }
+
+    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+        if self.ptr.is_none() {
+            return None;
+        }
+        let ptr = self.ptr.as_ref().unwrap();
+        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        Some(&*raw)
+    }
+}
+
+impl<'a, A> Iterator for LfuIter<'a, A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        unsafe {
+            let link_main = self.iter_main.get();
+            let link_tiny = self.iter_tiny.get();
+
+            let link = match (link_main, link_tiny) {
+                (None, None) => return None,
+                (Some(link_main), None) => {
+                    let link = link_main.raw();
+                    self.iter_main.prev();
+                    link
+                }
+                (None, Some(link_tiny)) => {
+                    let link = link_tiny.raw();
+                    self.iter_tiny.prev();
+                    link
+                }
+                (Some(link_main), Some(link_tiny)) => {
+                    // Eviction from tiny or main depending on whether the tiny handle woould be
+                    // admitted to main cachce. If it would be, evict from main cache, otherwise
+                    // from tiny cache.
+                    if self.lfu.admit_to_main(link_main.raw(), link_tiny.raw()) {
+                        let link = link_main.raw();
+                        self.iter_main.prev();
+                        link
+                    } else {
+                        let link = link_tiny.raw();
+                        self.iter_tiny.prev();
+                        link
+                    }
+                }
+            };
+            self.update_ptr(link);
+            self.ptr()
+        }
+    }
+}
+
+// unsafe impl `Send + Sync` for structs with `NonNull` usage
+
+unsafe impl<A> Send for Lfu<A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+unsafe impl<A> Sync for Lfu<A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+unsafe impl Send for LfuLink {}
+unsafe impl Sync for LfuLink {}
+
+unsafe impl<'a, A> Send for LfuIter<'a, A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+unsafe impl<'a, A> Sync for LfuIter<'a, A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+impl<A> EvictionPolicy<A> for Lfu<A>
+where
+    A: KeyAdapter<Link = LfuLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Link = LfuLink;
+
+    type Config = LfuConfig;
+
+    type E<'e> = LfuIter<'e, A>;
+
+    fn new(config: Self::Config) -> Self {
+        Self::new(config)
+    }
+
+    fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        self.insert(ptr)
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
+    ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
+        self.remove(ptr)
+    }
+
+    fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        self.access(ptr)
+    }
+
+    fn iter(&self) -> Self::E<'_> {
+        self.iter()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use itertools::Itertools;
+
+    use crate::key_adapter;
+
+    use super::*;
+
+    struct LfuItem {
+        link: LfuLink,
+        key: u64,
+    }
+
+    impl LfuItem {
+        fn new(key: u64) -> Self {
+            Self {
+                link: LfuLink::default(),
+                key,
+            }
+        }
+    }
+
+    intrusive_adapter! { LfuItemAdapter = Arc<LfuItem>: LfuItem { link: LfuLink } }
+    key_adapter! { LfuItemAdapter = LfuItem { key: u64 } }
+
+    #[test]
+    fn test_lfu_simple() {
+        let config = LfuConfig {
+            window_to_cache_size_ratio: 10,
+            tiny_lru_capacity_ratio: 0.01,
+        };
+        let mut lfu = Lfu::<LfuItemAdapter>::new(config);
+
+        let items = (0..101).map(LfuItem::new).map(Arc::new).collect_vec();
+        for item in items.iter().take(100) {
+            lfu.insert(item.clone());
+        }
+
+        assert_eq!(99, lfu.lru_main.len());
+        assert_eq!(1, lfu.lru_tiny.len());
+        assert_eq!(items[0].link.lru_type(), LruType::Tiny);
+
+        // 0 will be evicted at last because it is on tiny lru but its frequency equals to others
+        assert_eq!(
+            (1..100).chain([0].into_iter()).collect_vec(),
+            lfu.iter().map(|item| item.key).collect_vec()
+        );
+
+        for item in items.iter().take(100) {
+            lfu.access(item);
+        }
+        lfu.access(&items[0]);
+        lfu.insert(items[100].clone());
+
+        assert_eq!(items[0].link.lru_type(), LruType::Main);
+        assert_eq!(items[100].link.lru_type(), LruType::Tiny);
+
+        assert_eq!(
+            [100]
+                .into_iter()
+                .chain(1..100)
+                .chain([0].into_iter())
+                .collect_vec(),
+            lfu.iter().map(|item| item.key).collect_vec()
+        );
+
+        let to_remove_items = lfu.iter().cloned().collect_vec();
+        for item in to_remove_items {
+            lfu.remove(&item);
+        }
+
+        for item in items {
+            assert_eq!(Arc::strong_count(&item), 1);
+        }
+    }
+}
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
new file mode 100644
index 00000000..86584da5
--- /dev/null
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -0,0 +1,446 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{mem::ManuallyDrop, ptr::NonNull};
+
+use crate::{
+    collections::dlist::{DList, DListIter, DListLink},
+    core::{
+        adapter::{KeyAdapter, Link},
+        pointer::PointerOps,
+    },
+    intrusive_adapter,
+};
+
+use super::{Adapter, EvictionPolicy};
+
+#[derive(Clone, Debug)]
+pub struct LruConfig {
+    /// Insertion point of the new entry, between 0 and 1.
+    pub lru_insertion_point_fraction: f64,
+}
+
+#[derive(Debug, Default)]
+pub struct LruLink {
+    link_lru: DListLink,
+
+    is_in_tail: bool,
+}
+
+impl LruLink {
+    fn raw(&self) -> NonNull<Self> {
+        unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
+    }
+}
+
+impl Link for LruLink {
+    fn is_linked(&self) -> bool {
+        self.link_lru.is_linked()
+    }
+}
+
+intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DListLink } }
+
+pub struct Lru<A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    /// lru list
+    lru: DList<LruLinkAdapter>,
+
+    /// insertion point
+    insertion_point: Option<NonNull<LruLink>>,
+
+    /// length of tail after insertion point
+    tail_len: usize,
+
+    config: LruConfig,
+
+    adapter: A,
+}
+
+impl<A> Lru<A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    fn new(config: LruConfig) -> Self {
+        Self {
+            lru: DList::new(),
+
+            insertion_point: None,
+
+            tail_len: 0,
+
+            config,
+
+            adapter: A::new(),
+        }
+    }
+
+    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
+
+            assert!(!link.as_ref().is_linked());
+
+            self.insert_lru(link);
+
+            self.update_lru_insertion_point();
+        }
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<A::PointerOps as PointerOps>::Pointer,
+    ) -> <A::PointerOps as PointerOps>::Pointer {
+        unsafe {
+            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
+
+            assert!(link.as_ref().is_linked());
+
+            self.ensuer_not_insertion_point(link);
+            self.lru
+                .iter_mut_from_raw(link.as_ref().link_lru.raw())
+                .remove()
+                .unwrap();
+            if link.as_ref().is_in_tail {
+                link.as_mut().is_in_tail = false;
+                self.tail_len -= 1;
+            }
+
+            self.adapter.pointer_ops().from_raw(item)
+        }
+    }
+
+    fn access(&mut self, ptr: &<A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
+
+            assert!(link.as_ref().is_linked());
+
+            self.ensuer_not_insertion_point(link);
+
+            self.move_to_lru_front(link);
+
+            if link.as_ref().is_in_tail {
+                link.as_mut().is_in_tail = false;
+                self.tail_len -= 1;
+                self.update_lru_insertion_point();
+            }
+        }
+    }
+
+    fn iter(&self) -> LruIter<'_, A> {
+        let mut iter = self.lru.iter();
+        iter.tail();
+        LruIter {
+            iter,
+            lru: self,
+            ptr: ManuallyDrop::new(None),
+        }
+    }
+
+    fn update_lru_insertion_point(&mut self) {
+        unsafe {
+            if self.config.lru_insertion_point_fraction == 0.0 {
+                return;
+            }
+
+            if self.insertion_point.is_none() {
+                self.insertion_point = self.lru.back().map(LruLink::raw);
+                self.tail_len = 0;
+                if let Some(insertion_point) = &mut self.insertion_point {
+                    insertion_point.as_mut().is_in_tail = true;
+                    self.tail_len += 1;
+                }
+            }
+
+            if self.lru.len() <= 1 {
+                return;
+            }
+
+            assert!(self.insertion_point.is_some());
+
+            let expected_tail_len =
+                (self.lru.len() as f64 * (1.0 - self.config.lru_insertion_point_fraction)) as usize;
+
+            let mut curr = self.insertion_point.unwrap();
+            while self.tail_len < expected_tail_len
+                && Some(curr) != self.lru.front().map(LruLink::raw)
+            {
+                curr = self.lru_prev(curr).unwrap();
+                curr.as_mut().is_in_tail = true;
+                self.tail_len += 1;
+            }
+            while self.tail_len > expected_tail_len
+                && Some(curr) != self.lru.back().map(LruLink::raw)
+            {
+                curr.as_mut().is_in_tail = false;
+                self.tail_len -= 1;
+                curr = self.lru_next(curr).unwrap();
+            }
+
+            self.insertion_point = Some(curr);
+        }
+    }
+
+    unsafe fn ensuer_not_insertion_point(&mut self, link: NonNull<LruLink>) {
+        if Some(link) == self.insertion_point {
+            self.insertion_point = self.lru_prev(link);
+            match &mut self.insertion_point {
+                Some(insertion_point) => {
+                    self.tail_len += 1;
+                    insertion_point.as_mut().is_in_tail = true;
+                }
+                // TODO(MrCroxx): think ?
+                None => assert_eq!(self.lru.len(), 1),
+            }
+        }
+    }
+
+    unsafe fn insert_lru(&mut self, link: NonNull<LruLink>) {
+        match self.insertion_point {
+            Some(insertion_point) => self
+                .lru
+                .iter_mut_from_raw(insertion_point.as_ref().link_lru.raw())
+                .insert_before(link),
+            None => self.lru.push_front(link),
+        }
+    }
+
+    unsafe fn move_to_lru_front(&mut self, link: NonNull<LruLink>) {
+        self.lru
+            .iter_mut_from_raw(link.as_ref().link_lru.raw())
+            .remove()
+            .unwrap();
+        self.lru.push_front(link);
+    }
+
+    unsafe fn lru_prev(&self, link: NonNull<LruLink>) -> Option<NonNull<LruLink>> {
+        let mut iter = self.lru.iter_from_raw(link.as_ref().link_lru.raw());
+        iter.prev();
+        iter.get().map(LruLink::raw)
+    }
+
+    unsafe fn lru_next(&self, link: NonNull<LruLink>) -> Option<NonNull<LruLink>> {
+        let mut iter = self.lru.iter_from_raw(link.as_ref().link_lru.raw());
+        iter.next();
+        iter.get().map(LruLink::raw)
+    }
+}
+
+pub struct LruIter<'a, A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    lru: &'a Lru<A>,
+    iter: DListIter<'a, LruLinkAdapter>,
+
+    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+}
+
+impl<'a, A> LruIter<'a, A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    unsafe fn update_ptr(&mut self, link: NonNull<LruLink>) {
+        std::mem::forget(self.ptr.take());
+
+        let item = self.lru.adapter.link2item(link.as_ptr());
+        let ptr = self.lru.adapter.pointer_ops().from_raw(item);
+        self.ptr = ManuallyDrop::new(Some(ptr));
+    }
+
+    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+        if self.ptr.is_none() {
+            return None;
+        }
+        let ptr = self.ptr.as_ref().unwrap();
+        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        Some(&*raw)
+    }
+}
+
+impl<'a, A> Iterator for LruIter<'a, A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        unsafe {
+            let link = match self.iter.get() {
+                Some(link) => {
+                    let link = link.raw();
+                    self.iter.prev();
+                    link
+                }
+                None => return None,
+            };
+            self.update_ptr(link);
+            self.ptr()
+        }
+    }
+}
+
+// unsafe impl `Send + Sync` for structs with `NonNull` usage
+
+unsafe impl<A> Send for Lru<A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+unsafe impl<A> Sync for Lru<A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+unsafe impl Send for LruLink {}
+unsafe impl Sync for LruLink {}
+
+unsafe impl<'a, A> Send for LruIter<'a, A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+unsafe impl<'a, A> Sync for LruIter<'a, A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+impl<A> EvictionPolicy<A> for Lru<A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Link = LruLink;
+
+    type Config = LruConfig;
+
+    type E<'e> = LruIter<'e, A>;
+
+    fn new(config: Self::Config) -> Self {
+        Self::new(config)
+    }
+
+    fn insert(&mut self, ptr: <<A>::PointerOps as PointerOps>::Pointer) {
+        self.insert(ptr)
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<<A>::PointerOps as PointerOps>::Pointer,
+    ) -> <<A>::PointerOps as PointerOps>::Pointer {
+        self.remove(ptr)
+    }
+
+    fn access(&mut self, ptr: &<<A>::PointerOps as PointerOps>::Pointer) {
+        self.access(ptr)
+    }
+
+    fn iter(&self) -> Self::E<'_> {
+        self.iter()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use itertools::Itertools;
+
+    use crate::key_adapter;
+
+    use super::*;
+
+    struct LruItem {
+        link: LruLink,
+        key: u64,
+    }
+
+    impl LruItem {
+        fn new(key: u64) -> Self {
+            Self {
+                link: LruLink::default(),
+                key,
+            }
+        }
+    }
+
+    intrusive_adapter! { LruItemAdapter = Arc<LruItem>: LruItem { link: LruLink } }
+    key_adapter! { LruItemAdapter = LruItem { key: u64 } }
+
+    #[test]
+    fn test_lru_simple() {
+        let config = LruConfig {
+            lru_insertion_point_fraction: 0.0,
+        };
+        let mut lru = Lru::<LruItemAdapter>::new(config);
+
+        let handles = vec![
+            Arc::new(LruItem::new(0)),
+            Arc::new(LruItem::new(1)),
+            Arc::new(LruItem::new(2)),
+        ];
+
+        lru.insert(handles[0].clone());
+        lru.insert(handles[1].clone());
+        lru.insert(handles[2].clone());
+
+        assert_eq!(vec![0, 1, 2], lru.iter().map(|item| item.key).collect_vec());
+
+        lru.access(&handles[1]);
+
+        assert_eq!(vec![0, 2, 1], lru.iter().map(|item| item.key).collect_vec());
+
+        lru.remove(&handles[2]);
+
+        assert_eq!(vec![0, 1], lru.iter().map(|item| item.key).collect_vec());
+
+        lru.remove(&handles[0]);
+        lru.remove(&handles[1]);
+
+        for handle in handles {
+            assert_eq!(Arc::strong_count(&handle), 1);
+        }
+    }
+}
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
new file mode 100644
index 00000000..9dbd3f3a
--- /dev/null
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -0,0 +1,57 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use crate::core::adapter::{Adapter, KeyAdapter, Link};
+use crate::core::pointer::PointerOps;
+
+pub trait Config = Send + Sync + 'static;
+
+pub trait EvictionPolicy<A>: Send + Sync + 'static
+where
+    A: KeyAdapter<Link = Self::Link>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Link: Link;
+    type Config: Config;
+    type E<'e>: Iterator<Item = &'e <A::PointerOps as PointerOps>::Pointer>;
+
+    fn new(config: Self::Config) -> Self;
+
+    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer);
+
+    fn remove(
+        &mut self,
+        ptr: &<A::PointerOps as PointerOps>::Pointer,
+    ) -> <A::PointerOps as PointerOps>::Pointer;
+
+    fn access(&mut self, ptr: &<A::PointerOps as PointerOps>::Pointer);
+
+    fn iter(&self) -> Self::E<'_>;
+
+    fn push(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        self.insert(ptr)
+    }
+
+    fn pop(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+        let ptr = {
+            let mut iter = self.iter();
+            let ptr = iter.next();
+            ptr.cloned()
+        };
+        ptr.map(|ptr| self.remove(&ptr))
+    }
+}
+
+pub mod lfu;
+pub mod lru;
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
new file mode 100644
index 00000000..af07e036
--- /dev/null
+++ b/foyer-intrusive/src/lib.rs
@@ -0,0 +1,56 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(associated_type_bounds)]
+#![feature(ptr_metadata)]
+#![feature(trait_alias)]
+#![allow(clippy::new_without_default)]
+#![allow(clippy::wrong_self_convention)]
+#![allow(clippy::vtable_address_comparisons)]
+
+pub use memoffset::offset_of;
+
+/// Unsafe macro to get a raw pointer to an outer object from a pointer to one
+/// of its fields.
+///
+/// # Examples
+///
+/// ```
+/// use foyer_intrusive::container_of;
+///
+/// struct S { x: u32, y: u32 };
+/// let container = S { x: 1, y: 2 };
+/// let field = &container.x;
+/// let container2: *const S = unsafe { container_of!(field, S, x) };
+/// assert_eq!(&container as *const S, container2);
+/// ```
+///
+/// # Safety
+///
+/// This is unsafe because it assumes that the given expression is a valid
+/// pointer to the specified field of some container type.
+#[macro_export]
+macro_rules! container_of {
+    ($ptr:expr, $container:path, $field:ident) => {
+        #[allow(clippy::cast_ptr_alignment)]
+        {
+            ($ptr as *const _ as *const u8).sub($crate::offset_of!($container, $field))
+                as *const $container
+        }
+    };
+}
+
+pub mod collections;
+pub mod core;
+pub mod eviction;
diff --git a/foyer-policy/Cargo.toml b/foyer-policy/Cargo.toml
new file mode 100644
index 00000000..3c46e3af
--- /dev/null
+++ b/foyer-policy/Cargo.toml
@@ -0,0 +1,42 @@
+[package]
+name = "foyer-policy"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1"
+bytes = "1"
+cmsketch = "0.1"
+crossbeam = "0.8"
+foyer-utils = { path = "../foyer-utils" }
+futures = "0.3"
+itertools = "0.10.5"
+libc = "0.2"
+memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+paste = "1.0"
+prometheus = "0.13"
+rand = "0.8.5"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+tracing = "0.1"
+twox-hash = "1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+hdrhistogram = "7"
+rand_mt = "4.2.1"
+tempfile = "3"
diff --git a/foyer/src/policies/lru.rs b/foyer-policy/src/eviction/lru.rs
similarity index 83%
rename from foyer/src/policies/lru.rs
rename to foyer-policy/src/eviction/lru.rs
index ebac9999..8e542311 100644
--- a/foyer/src/policies/lru.rs
+++ b/foyer-policy/src/eviction/lru.rs
@@ -32,7 +32,7 @@ use std::time::SystemTime;
 use foyer_utils::dlist::{DList, Entry, Iter};
 use foyer_utils::intrusive_dlist;
 
-use super::Index;
+use super::Item;
 
 #[derive(Clone, Debug)]
 pub struct Config {
@@ -40,7 +40,7 @@ pub struct Config {
     pub lru_insertion_point_fraction: f64,
 }
 
-pub struct Handle<I: Index> {
+pub struct Handle<T: Item> {
     entry: Entry,
 
     is_in_cache: bool,
@@ -49,11 +49,11 @@ pub struct Handle<I: Index> {
 
     is_in_tail: bool,
 
-    index: I,
+    item: T,
 }
 
-impl<I: Index> Handle<I> {
-    fn new(index: I) -> Self {
+impl<T: Item> Handle<T> {
+    fn new(item: T) -> Self {
         Self {
             entry: Entry::default(),
 
@@ -63,23 +63,19 @@ impl<I: Index> Handle<I> {
 
             is_in_tail: false,
 
-            index,
+            item,
         }
     }
-
-    fn index(&self) -> &I {
-        &self.index
-    }
 }
 
-intrusive_dlist! { Handle<I: Index>, entry, HandleDListAdapter}
+intrusive_dlist! { Handle<T: Item>, entry, HandleDListAdapter}
 
-pub struct Lru<I: Index> {
+pub struct Lru<T: Item> {
     /// lru list
-    lru: DList<Handle<I>, HandleDListAdapter>,
+    lru: DList<Handle<T>, HandleDListAdapter>,
 
     /// insertion point
-    insertion_point: Option<NonNull<Handle<I>>>,
+    insertion_point: Option<NonNull<Handle<T>>>,
 
     /// length of tail after insertion point
     tail_len: usize,
@@ -87,7 +83,7 @@ pub struct Lru<I: Index> {
     config: Config,
 }
 
-impl<I: Index> Lru<I> {
+impl<T: Item> Lru<T> {
     fn new(config: Config) -> Self {
         Self {
             lru: DList::new(),
@@ -102,7 +98,7 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
     /// returns `false` otherwise.
-    fn access(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn access(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
         unsafe {
             handle.as_mut().is_accessed = true;
 
@@ -127,7 +123,7 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if handle is successfully added into the lru,
     /// returns `false` if the handle is already in the lru.
-    fn insert(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn insert(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
         unsafe {
             if handle.as_ref().is_in_cache {
                 return false;
@@ -148,7 +144,7 @@ impl<I: Index> Lru<I> {
 
     /// Returns `true` if handle is successfully removed from the lru,
     /// returns `false` if the handle is unchanged.
-    fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn remove(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
         unsafe {
             if !handle.as_ref().is_in_cache {
                 return false;
@@ -166,7 +162,7 @@ impl<I: Index> Lru<I> {
         }
     }
 
-    fn eviction_iter(&self) -> EvictionIter<'_, I> {
+    fn eviction_iter(&self) -> EvictionIter<'_, T> {
         unsafe {
             let mut iter = self.lru.iter();
             iter.tail();
@@ -214,7 +210,7 @@ impl<I: Index> Lru<I> {
         }
     }
 
-    fn ensuer_not_insertion_point(&mut self, handle: NonNull<Handle<I>>) {
+    fn ensuer_not_insertion_point(&mut self, handle: NonNull<Handle<T>>) {
         unsafe {
             if Some(handle) == self.insertion_point {
                 self.insertion_point = self.lru.prev(handle);
@@ -231,19 +227,19 @@ impl<I: Index> Lru<I> {
     }
 }
 
-pub struct EvictionIter<'a, I: Index> {
-    iter: Iter<'a, Handle<I>, HandleDListAdapter>,
+pub struct EvictionIter<'a, T: Item> {
+    iter: Iter<'a, Handle<T>, HandleDListAdapter>,
 }
 
-impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
-    type Item = &'a I;
+impl<'a, T: Item> Iterator for EvictionIter<'a, T> {
+    type Item = &'a T;
 
     fn next(&mut self) -> Option<Self::Item> {
         unsafe {
             match self.iter.element() {
                 Some(element) => {
                     self.iter.prev();
-                    Some(&element.as_ref().index)
+                    Some(&element.as_ref().item)
                 }
                 None => None,
             }
@@ -253,34 +249,34 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
 
 // unsafe impl `Send + Sync` for structs with `NonNull` usage
 
-unsafe impl<I: Index> Send for Lru<I> {}
-unsafe impl<I: Index> Sync for Lru<I> {}
+unsafe impl<T: Item> Send for Lru<T> {}
+unsafe impl<T: Item> Sync for Lru<T> {}
 
-unsafe impl<I: Index> Send for Handle<I> {}
-unsafe impl<I: Index> Sync for Handle<I> {}
+unsafe impl<T: Item> Send for Handle<T> {}
+unsafe impl<T: Item> Sync for Handle<T> {}
 
-unsafe impl<'a, I: Index> Send for EvictionIter<'a, I> {}
-unsafe impl<'a, I: Index> Sync for EvictionIter<'a, I> {}
+unsafe impl<'a, T: Item> Send for EvictionIter<'a, T> {}
+unsafe impl<'a, T: Item> Sync for EvictionIter<'a, T> {}
 
 impl super::Config for Config {}
 
-impl<I: Index> super::Handle for Handle<I> {
-    type I = I;
+impl<T: Item> super::Handle for Handle<T> {
+    type T = T;
 
-    fn new(index: Self::I) -> Self {
-        Self::new(index)
+    fn new(item: Self::T) -> Self {
+        Self::new(item)
     }
 
-    fn index(&self) -> &Self::I {
-        self.index()
+    fn item(&self) -> &Self::T {
+        &self.item
     }
 }
 
-impl<I: Index> super::Policy for Lru<I> {
-    type I = I;
+impl<T: Item> super::Policy for Lru<T> {
+    type T = T;
     type C = Config;
-    type H = Handle<I>;
-    type E<'e> = EvictionIter<'e, I>;
+    type H = Handle<T>;
+    type E<'e> = EvictionIter<'e, T>;
 
     fn new(config: Self::C) -> Self {
         Lru::new(config)
@@ -309,7 +305,7 @@ mod tests {
 
     use super::*;
 
-    fn ptr<I: Index>(handle: &mut Handle<I>) -> NonNull<Handle<I>> {
+    fn ptr<T: Item>(handle: &mut Handle<T>) -> NonNull<Handle<T>> {
         unsafe { NonNull::new_unchecked(handle as *mut _) }
     }
 
diff --git a/foyer/src/policies/mod.rs b/foyer-policy/src/eviction/mod.rs
similarity index 86%
rename from foyer/src/policies/mod.rs
rename to foyer-policy/src/eviction/mod.rs
index 475640c5..cbcf6eac 100644
--- a/foyer/src/policies/mod.rs
+++ b/foyer-policy/src/eviction/mod.rs
@@ -20,14 +20,15 @@
 pub mod lru;
 pub mod tinylfu;
 
-use crate::Index;
 use std::ptr::NonNull;
 
+use crate::Item;
+
 pub trait Policy: Send + Sync + 'static {
-    type I: Index;
+    type T: Item;
     type C: Config;
-    type H: Handle<I = Self::I>;
-    type E<'e>: Iterator<Item = &'e Self::I>;
+    type H: Handle<T = Self::T>;
+    type E<'e>: Iterator<Item = &'e Self::T>;
 
     fn new(config: Self::C) -> Self;
 
@@ -43,9 +44,9 @@ pub trait Policy: Send + Sync + 'static {
 pub trait Config: Send + Sync + std::fmt::Debug + Clone + 'static {}
 
 pub trait Handle: Send + Sync + 'static {
-    type I: Index;
+    type T: Item;
 
-    fn new(index: Self::I) -> Self;
+    fn new(index: Self::T) -> Self;
 
-    fn index(&self) -> &Self::I;
+    fn item(&self) -> &Self::T;
 }
diff --git a/foyer/src/policies/tinylfu.rs b/foyer-policy/src/eviction/tinylfu.rs
similarity index 86%
rename from foyer/src/policies/tinylfu.rs
rename to foyer-policy/src/eviction/tinylfu.rs
index e8e44144..5f75fd37 100644
--- a/foyer/src/policies/tinylfu.rs
+++ b/foyer-policy/src/eviction/tinylfu.rs
@@ -36,7 +36,7 @@ use twox_hash::XxHash64;
 use foyer_utils::dlist::{DList, Entry, Iter};
 use foyer_utils::intrusive_dlist;
 
-use super::Index;
+use super::Item;
 
 const MIN_CAPACITY: usize = 100;
 const ERROR_THRESHOLD: f64 = 5.0;
@@ -58,7 +58,7 @@ enum LruType {
     Main,
 }
 
-pub struct Handle<I: Index> {
+pub struct Handle<T: Item> {
     entry_tiny: Entry,
     entry_main: Entry,
 
@@ -68,11 +68,11 @@ pub struct Handle<I: Index> {
 
     lru_type: LruType,
 
-    index: I,
+    item: T,
 }
 
-impl<I: Index> Handle<I> {
-    fn new(index: I) -> Self {
+impl<T: Item> Handle<T> {
+    fn new(item: T) -> Self {
         Self {
             entry_tiny: Entry::default(),
             entry_main: Entry::default(),
@@ -83,17 +83,13 @@ impl<I: Index> Handle<I> {
 
             lru_type: LruType::Tiny,
 
-            index,
+            item,
         }
     }
-
-    fn index(&self) -> &I {
-        &self.index
-    }
 }
 
-intrusive_dlist! { Handle<I: Index>, entry_tiny, HandleDListTinyAdapter}
-intrusive_dlist! { Handle<I: Index>, entry_main, HandleDListMainAdapter}
+intrusive_dlist! { Handle<T: Item>, entry_tiny, HandleDListTinyAdapter}
+intrusive_dlist! { Handle<T: Item>, entry_main, HandleDListMainAdapter}
 
 /// Implements the W-TinyLFU cache eviction policy as described in -
 ///
@@ -122,12 +118,12 @@ intrusive_dlist! { Handle<I: Index>, entry_main, HandleDListMainAdapter}
 ///
 /// Tiny cache size:
 /// This default to 1%. There's no need to tune this parameter.
-pub struct TinyLfu<I: Index> {
+pub struct TinyLfu<T: Item> {
     /// tiny lru list
-    lru_tiny: DList<Handle<I>, HandleDListTinyAdapter>,
+    lru_tiny: DList<Handle<T>, HandleDListTinyAdapter>,
 
     /// main lru list
-    lru_main: DList<Handle<I>, HandleDListMainAdapter>,
+    lru_main: DList<Handle<T>, HandleDListMainAdapter>,
 
     /// the window length counter
     window_size: usize,
@@ -146,7 +142,7 @@ pub struct TinyLfu<I: Index> {
     config: Config,
 }
 
-impl<I: Index> TinyLfu<I> {
+impl<T: Item> TinyLfu<T> {
     pub fn new(config: Config) -> Self {
         let mut res = Self {
             lru_tiny: DList::new(),
@@ -167,7 +163,7 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
     /// returns `false` otherwise.
-    fn access(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn access(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
         unsafe {
             handle.as_mut().is_accessed = true;
 
@@ -188,7 +184,7 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if handle is successfully added into the lru,
     /// returns `false` if the handle is already in the lru.
-    fn insert(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn insert(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
         unsafe {
             if handle.as_ref().is_in_cache {
                 return false;
@@ -227,7 +223,7 @@ impl<I: Index> TinyLfu<I> {
 
     /// Returns `true` if handle is successfully removed from the lru,
     /// returns `false` if the handle is unchanged.
-    fn remove(&mut self, mut handle: NonNull<Handle<I>>) -> bool {
+    fn remove(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
         unsafe {
             if !handle.as_ref().is_in_cache {
                 return false;
@@ -245,7 +241,7 @@ impl<I: Index> TinyLfu<I> {
         }
     }
 
-    fn eviction_iter<'a>(&'a self) -> EvictionIter<'a, I> {
+    fn eviction_iter<'a>(&'a self) -> EvictionIter<'a, T> {
         unsafe {
             let mut iter_main: Iter<'a, _, _> = self.lru_main.iter();
             iter_main.tail();
@@ -283,7 +279,7 @@ impl<I: Index> TinyLfu<I> {
         self.frequencies = CMSketchUsize::new_with_size(num_counters, HASH_COUNT);
     }
 
-    fn update_frequencies(&mut self, handle: NonNull<Handle<I>>) {
+    fn update_frequencies(&mut self, handle: NonNull<Handle<T>>) {
         self.frequencies.record(Self::hash_handle(handle));
         self.window_size += 1;
 
@@ -327,8 +323,8 @@ impl<I: Index> TinyLfu<I> {
 
     fn admit_to_main(
         &self,
-        handle_main: NonNull<Handle<I>>,
-        handle_tiny: NonNull<Handle<I>>,
+        handle_main: NonNull<Handle<T>>,
+        handle_tiny: NonNull<Handle<T>>,
     ) -> bool {
         unsafe {
             assert_eq!(handle_main.as_ref().lru_type, LruType::Main);
@@ -341,21 +337,21 @@ impl<I: Index> TinyLfu<I> {
         }
     }
 
-    fn hash_handle(handle: NonNull<Handle<I>>) -> u64 {
+    fn hash_handle(handle: NonNull<Handle<T>>) -> u64 {
         let mut hasher = XxHash64::default();
-        unsafe { handle.as_ref().index.hash(&mut hasher) };
+        unsafe { handle.as_ref().item.hash(&mut hasher) };
         hasher.finish()
     }
 }
 
-pub struct EvictionIter<'a, I: Index> {
-    tinylfu: &'a TinyLfu<I>,
-    iter_main: Iter<'a, Handle<I>, HandleDListMainAdapter>,
-    iter_tiny: Iter<'a, Handle<I>, HandleDListTinyAdapter>,
+pub struct EvictionIter<'a, T: Item> {
+    tinylfu: &'a TinyLfu<T>,
+    iter_main: Iter<'a, Handle<T>, HandleDListMainAdapter>,
+    iter_tiny: Iter<'a, Handle<T>, HandleDListTinyAdapter>,
 }
 
-impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
-    type Item = &'a I;
+impl<'a, T: Item> Iterator for EvictionIter<'a, T> {
+    type Item = &'a T;
 
     fn next(&mut self) -> Option<Self::Item> {
         unsafe {
@@ -366,11 +362,11 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
                 (None, None) => None,
                 (Some(handle_main), None) => {
                     self.iter_main.prev();
-                    Some(&handle_main.as_ref().index)
+                    Some(&handle_main.as_ref().item)
                 }
                 (None, Some(handle_tiny)) => {
                     self.iter_tiny.prev();
-                    Some(&handle_tiny.as_ref().index)
+                    Some(&handle_tiny.as_ref().item)
                 }
                 (Some(handle_main), Some(handle_tiny)) => {
                     // Eviction from tiny or main depending on whether the tiny handle woould be
@@ -378,10 +374,10 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
                     // from tiny cache.
                     if self.tinylfu.admit_to_main(handle_main, handle_tiny) {
                         self.iter_main.prev();
-                        Some(&handle_main.as_ref().index)
+                        Some(&handle_main.as_ref().item)
                     } else {
                         self.iter_tiny.prev();
-                        Some(&handle_tiny.as_ref().index)
+                        Some(&handle_tiny.as_ref().item)
                     }
                 }
             }
@@ -391,34 +387,34 @@ impl<'a, I: Index> Iterator for EvictionIter<'a, I> {
 
 // unsafe impl `Send + Sync` for structs with `NonNull` usage
 
-unsafe impl<I: Index> Send for TinyLfu<I> {}
-unsafe impl<I: Index> Sync for TinyLfu<I> {}
+unsafe impl<T: Item> Send for TinyLfu<T> {}
+unsafe impl<T: Item> Sync for TinyLfu<T> {}
 
-unsafe impl<I: Index> Send for Handle<I> {}
-unsafe impl<I: Index> Sync for Handle<I> {}
+unsafe impl<T: Item> Send for Handle<T> {}
+unsafe impl<T: Item> Sync for Handle<T> {}
 
-unsafe impl<'a, I: Index> Send for EvictionIter<'a, I> {}
-unsafe impl<'a, I: Index> Sync for EvictionIter<'a, I> {}
+unsafe impl<'a, T: Item> Send for EvictionIter<'a, T> {}
+unsafe impl<'a, T: Item> Sync for EvictionIter<'a, T> {}
 
 impl super::Config for Config {}
 
-impl<I: Index> super::Handle for Handle<I> {
-    type I = I;
+impl<T: Item> super::Handle for Handle<T> {
+    type T = T;
 
-    fn new(index: Self::I) -> Self {
+    fn new(index: Self::T) -> Self {
         Self::new(index)
     }
 
-    fn index(&self) -> &Self::I {
-        self.index()
+    fn item(&self) -> &Self::T {
+        &self.item
     }
 }
 
-impl<I: Index> super::Policy for TinyLfu<I> {
-    type I = I;
+impl<T: Item> super::Policy for TinyLfu<T> {
+    type T = T;
     type C = Config;
-    type H = Handle<I>;
-    type E<'e> = EvictionIter<'e, I>;
+    type H = Handle<T>;
+    type E<'e> = EvictionIter<'e, T>;
 
     fn new(config: Self::C) -> Self {
         TinyLfu::new(config)
@@ -447,7 +443,7 @@ mod tests {
 
     use super::*;
 
-    fn ptr<I: Index>(handle: &mut Handle<I>) -> NonNull<Handle<I>> {
+    fn ptr<T: Item>(handle: &mut Handle<T>) -> NonNull<Handle<T>> {
         unsafe { NonNull::new_unchecked(handle as *mut _) }
     }
 
diff --git a/foyer-policy/src/lib.rs b/foyer-policy/src/lib.rs
new file mode 100644
index 00000000..4dddce56
--- /dev/null
+++ b/foyer-policy/src/lib.rs
@@ -0,0 +1,30 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(trait_alias)]
+#![feature(pattern)]
+
+pub mod eviction;
+pub mod reinsertion;
+
+pub trait Item = PartialOrd
+    + Ord
+    + PartialEq
+    + Eq
+    + Clone
+    + std::hash::Hash
+    + Send
+    + Sync
+    + 'static
+    + std::fmt::Debug;
diff --git a/foyer-policy/src/reinsertion/mod.rs b/foyer-policy/src/reinsertion/mod.rs
new file mode 100644
index 00000000..9717a1ff
--- /dev/null
+++ b/foyer-policy/src/reinsertion/mod.rs
@@ -0,0 +1,29 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+pub trait Policy {}
diff --git a/foyer-utils/src/dlist.rs b/foyer-utils/src/dlist.rs
index a134ca84..7f7a9d44 100644
--- a/foyer-utils/src/dlist.rs
+++ b/foyer-utils/src/dlist.rs
@@ -25,6 +25,12 @@ pub struct Entry {
     next: Option<NonNull<Entry>>,
 }
 
+impl Entry {
+    pub fn is_linked(&self) -> bool {
+        self.prev.is_some() || self.next.is_some()
+    }
+}
+
 pub trait Adapter<E> {
     /// entry ptr to element ptr
     fn en2el(_: NonNull<Entry>) -> NonNull<E>;
@@ -62,6 +68,8 @@ macro_rules! intrusive_dlist {
     };
 }
 
+pub use crate::intrusive_dlist;
+
 /// TODO: write docs
 #[derive(Debug)]
 pub struct DList<E, A: Adapter<E>> {
@@ -322,8 +330,6 @@ impl<E, A: Adapter<E>> DList<E, A> {
     }
 }
 
-pub trait DListExt {}
-
 pub struct Iter<'a, E, A: Adapter<E>> {
     dlist: &'a DList<E, A>,
     entry: Option<NonNull<Entry>>,
diff --git a/foyer-utils/src/lib.rs b/foyer-utils/src/lib.rs
index 2d58d97e..6c953bb6 100644
--- a/foyer-utils/src/lib.rs
+++ b/foyer-utils/src/lib.rs
@@ -16,3 +16,4 @@
 
 pub mod bits;
 pub mod dlist;
+pub mod queue;
diff --git a/foyer-utils/src/queue.rs b/foyer-utils/src/queue.rs
new file mode 100644
index 00000000..d2e0a2ae
--- /dev/null
+++ b/foyer-utils/src/queue.rs
@@ -0,0 +1,64 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::{fmt::Debug, sync::atomic::AtomicUsize};
+
+use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+use tokio::sync::Mutex;
+
+#[derive(Debug, Clone)]
+pub struct AsyncQueue<T: Debug> {
+    tx: UnboundedSender<T>,
+    rx: Arc<Mutex<UnboundedReceiver<T>>>,
+
+    size: Arc<AtomicUsize>,
+}
+
+impl<T: Debug> Default for AsyncQueue<T> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<T: Debug> AsyncQueue<T> {
+    pub fn new() -> Self {
+        let (tx, rx) = unbounded_channel();
+        Self {
+            tx,
+            rx: Arc::new(Mutex::new(rx)),
+            size: Arc::new(AtomicUsize::new(0)),
+        }
+    }
+
+    pub async fn acquire(&self) -> T {
+        let mut rx = self.rx.lock().await;
+        let item = rx.recv().await.unwrap();
+        self.size.fetch_sub(1, Ordering::Relaxed);
+        item
+    }
+
+    pub fn release(&self, item: T) {
+        self.tx.send(item).unwrap();
+    }
+
+    pub fn len(&self) -> usize {
+        self.size.load(Ordering::Relaxed)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 949daa8e..e33dfddc 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -12,6 +12,8 @@ async-trait = "0.1"
 bytes = "1"
 cmsketch = "0.1"
 crossbeam = "0.8"
+foyer-common = { path = "../foyer-common" }
+foyer-policy = { path = "../foyer-policy" }
 foyer-utils = { path = "../foyer-utils" }
 futures = "0.3"
 itertools = "0.10.5"
diff --git a/foyer/src/container.rs b/foyer/src/container.rs
index e3a91b59..7f13880d 100644
--- a/foyer/src/container.rs
+++ b/foyer/src/container.rs
@@ -23,9 +23,9 @@ use itertools::Itertools;
 use tokio::sync::{Mutex, MutexGuard};
 use twox_hash::XxHash64;
 
-use crate::policies::{Handle, Policy};
 use crate::store::Store;
 use crate::{Data, Index, Metrics, WrappedNonNull};
+use foyer_policy::eviction::{Handle, Policy};
 
 // TODO(MrCroxx): wrap own result type
 use crate::store::error::Result;
@@ -33,8 +33,8 @@ use crate::store::error::Result;
 pub struct Config<I, P, H, S>
 where
     I: Index,
-    P: Policy<I = I, H = H>,
-    H: Handle<I = I>,
+    P: Policy<T = I, H = H>,
+    H: Handle<T = I>,
     S: Store<I = I>,
 {
     pub capacity: usize,
@@ -49,8 +49,8 @@ where
 impl<I, P, H, S> std::fmt::Debug for Config<I, P, H, S>
 where
     I: Index,
-    P: Policy<I = I, H = H>,
-    H: Handle<I = I>,
+    P: Policy<T = I, H = H>,
+    H: Handle<T = I>,
     S: Store<I = I>,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -67,8 +67,8 @@ where
 pub struct Container<I, P, H, D, S>
 where
     I: Index,
-    P: Policy<I = I, H = H>,
-    H: Handle<I = I>,
+    P: Policy<T = I, H = H>,
+    H: Handle<T = I>,
     D: Data,
     S: Store<I = I, D = D>,
 {
@@ -81,8 +81,8 @@ where
 impl<I, P, H, D, S> Container<I, P, H, D, S>
 where
     I: Index,
-    P: Policy<I = I, H = H>,
-    H: Handle<I = I>,
+    P: Policy<T = I, H = H>,
+    H: Handle<T = I>,
     D: Data,
 
     S: Store<I = I, D = D>,
@@ -198,8 +198,8 @@ where
 struct Pool<I, P, H, D, S>
 where
     I: Index,
-    P: Policy<I = I, H = H>,
-    H: Handle<I = I>,
+    P: Policy<T = I, H = H>,
+    H: Handle<T = I>,
     D: Data,
     S: Store<I = I, D = D>,
 {
@@ -221,8 +221,8 @@ where
 impl<I, P, H, D, S> Pool<I, P, H, D, S>
 where
     I: Index,
-    P: Policy<I = I, H = H>,
-    H: Handle<I = I>,
+    P: Policy<T = I, H = H>,
+    H: Handle<T = I>,
     D: Data,
     S: Store<I = I, D = D>,
 {
@@ -282,8 +282,8 @@ where
 impl<I, P, H, D, S> Drop for Pool<I, P, H, D, S>
 where
     I: Index,
-    P: Policy<I = I, H = H>,
-    H: Handle<I = I>,
+    P: Policy<T = I, H = H>,
+    H: Handle<T = I>,
     D: Data,
     S: Store<I = I, D = D>,
 {
@@ -299,7 +299,7 @@ where
 struct PoolHandle<I, H>
 where
     I: Index,
-    H: Handle<I = I>,
+    H: Handle<T = I>,
 {
     weight: usize,
     // Use `WrappedNonNull` for ptrs that needs to cross .await points.
@@ -310,10 +310,10 @@ where
 mod tests {
     use super::*;
 
-    use crate::policies::lru::{Config as LruConfig, Handle as LruHandle, Lru};
-    use crate::policies::tinylfu::Handle as TinyLfuHandle;
     use crate::store::tests::MemoryStore;
     use crate::tests::is_send_sync_static;
+    use foyer_policy::eviction::lru::{Config as LruConfig, Handle as LruHandle, Lru};
+    use foyer_policy::eviction::tinylfu::Handle as TinyLfuHandle;
 
     #[tokio::test]
     async fn test_container_simple() {
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 3ef71eee..5637c07e 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -17,13 +17,10 @@
 
 use std::{fmt::Debug, ptr::NonNull};
 
-pub use policies::Policy;
-
 use paste::paste;
 
 mod container;
 mod metrics;
-mod policies;
 mod store;
 
 pub trait Weight {
@@ -92,38 +89,38 @@ pub type Error = store::error::Error;
 
 pub type TinyLfuReadOnlyFileStoreCache<I, D> = container::Container<
     I,
-    policies::tinylfu::TinyLfu<I>,
-    policies::tinylfu::Handle<I>,
+    foyer_policy::eviction::tinylfu::TinyLfu<I>,
+    foyer_policy::eviction::tinylfu::Handle<I>,
     D,
     store::read_only_file_store::ReadOnlyFileStore<I, D>,
 >;
 
 pub type TinyLfuReadOnlyFileStoreCacheConfig<I, D> = container::Config<
     I,
-    policies::tinylfu::TinyLfu<I>,
-    policies::tinylfu::Handle<I>,
+    foyer_policy::eviction::tinylfu::TinyLfu<I>,
+    foyer_policy::eviction::tinylfu::Handle<I>,
     store::read_only_file_store::ReadOnlyFileStore<I, D>,
 >;
 
 pub type LruReadOnlyFileStoreCache<I, D> = container::Container<
     I,
-    policies::lru::Lru<I>,
-    policies::lru::Handle<I>,
+    foyer_policy::eviction::lru::Lru<I>,
+    foyer_policy::eviction::lru::Handle<I>,
     D,
     store::read_only_file_store::ReadOnlyFileStore<I, D>,
 >;
 
 pub type LruReadOnlyFileStoreCacheConfig<I, D> = container::Config<
     I,
-    policies::lru::Lru<I>,
-    policies::lru::Handle<I>,
+    foyer_policy::eviction::lru::Lru<I>,
+    foyer_policy::eviction::lru::Handle<I>,
     store::read_only_file_store::ReadOnlyFileStore<I, D>,
 >;
 
 pub use metrics::Metrics;
 
-pub use policies::lru::Config as LruConfig;
-pub use policies::tinylfu::Config as TinyLfuConfig;
+pub use foyer_policy::eviction::lru::Config as LruConfig;
+pub use foyer_policy::eviction::tinylfu::Config as TinyLfuConfig;
 pub use store::read_only_file_store::Config as ReadOnlyFileStoreConfig;
 
 #[cfg(test)]
diff --git a/foyer/src/store/error.rs b/foyer/src/store/error.rs
index 81f26d36..c109bb55 100644
--- a/foyer/src/store/error.rs
+++ b/foyer/src/store/error.rs
@@ -18,10 +18,6 @@ pub enum Error {
     Io(#[from] std::io::Error),
     #[error("nix error: {0}")]
     Nix(#[from] nix::errno::Errno),
-    #[error("unsupported file system, super block magic: {0}")]
-    UnsupportedFilesystem(i64),
-    #[error("invalid slot: {0}")]
-    InvalidSlot(usize),
     #[error("other error: {0}")]
     Other(String),
 }

From e9714c572c7120609a947651e82132dfffe9b67d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 30 Jun 2023 12:26:43 +0800
Subject: [PATCH 020/261] feat: introduce FTL-like storage engine (#22)

* feat: introduce FTL-like storage engine

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* sort cargo file

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix memory leak

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/asan.yml                |   0
 .github/template/template.yml            |  15 +-
 .github/workflows/main.yml               |  13 +-
 .github/workflows/pull-request.yml       |  13 +-
 Cargo.lock                               | 204 ++++++++-
 Cargo.toml                               |   2 +
 foyer-bench/src/analyze.rs               |  12 +-
 foyer-bench/src/main.rs                  |  17 +-
 foyer-bench/src/utils.rs                 |   3 +-
 foyer-common/src/lib.rs                  |   3 +-
 foyer-intrusive/Cargo.toml               |   1 +
 foyer-intrusive/src/collections/dlist.rs |  47 ++-
 foyer-intrusive/src/core/adapter.rs      |  49 ++-
 foyer-intrusive/src/core/pointer.rs      |  40 +-
 foyer-intrusive/src/eviction/lfu.rs      |  46 +-
 foyer-intrusive/src/eviction/lru.rs      |  22 +-
 foyer-intrusive/src/eviction/mod.rs      |  10 +-
 foyer-policy/src/eviction/lru.rs         |   9 +-
 foyer-policy/src/eviction/tinylfu.rs     |  10 +-
 foyer-storage-bench/Cargo.toml           |  33 ++
 foyer-storage-bench/src/analyze.rs       | 278 +++++++++++++
 foyer-storage-bench/src/main.rs          | 386 +++++++++++++++++
 foyer-storage-bench/src/rate.rs          |  59 +++
 foyer-storage-bench/src/utils.rs         | 170 ++++++++
 foyer-storage/Cargo.toml                 |  46 ++
 foyer-storage/src/admission.rs           |  39 ++
 foyer-storage/src/device/error.rs        |  31 ++
 foyer-storage/src/device/fs.rs           | 266 ++++++++++++
 foyer-storage/src/device/io_buffer.rs    |  73 ++++
 foyer-storage/src/device/mod.rs          | 155 +++++++
 foyer-storage/src/error.rs               |  33 ++
 foyer-storage/src/flusher.rs             | 181 ++++++++
 foyer-storage/src/indices.rs             | 110 +++++
 foyer-storage/src/lib.rs                 |  31 ++
 foyer-storage/src/reclaimer.rs           | 178 ++++++++
 foyer-storage/src/region.rs              | 509 +++++++++++++++++++++++
 foyer-storage/src/region_manager.rs      | 181 ++++++++
 foyer-storage/src/reinsertion.rs         |  45 ++
 foyer-storage/src/slice.rs               |  97 +++++
 foyer-storage/src/store.rs               | 274 ++++++++++++
 foyer-utils/src/bits.rs                  |   6 +-
 foyer-utils/src/dlist.rs                 |   3 +-
 foyer-utils/src/queue.rs                 |  24 +-
 foyer/src/container.rs                   |  18 +-
 foyer/src/lib.rs                         |   3 +-
 foyer/src/store/file.rs                  |  23 +-
 foyer/src/store/mod.rs                   |   3 +-
 foyer/src/store/read_only_file_store.rs  |  32 +-
 rustfmt.toml                             |   1 +
 49 files changed, 3652 insertions(+), 152 deletions(-)
 delete mode 100644 .github/template/asan.yml
 create mode 100644 foyer-storage-bench/Cargo.toml
 create mode 100644 foyer-storage-bench/src/analyze.rs
 create mode 100644 foyer-storage-bench/src/main.rs
 create mode 100644 foyer-storage-bench/src/rate.rs
 create mode 100644 foyer-storage-bench/src/utils.rs
 create mode 100644 foyer-storage/Cargo.toml
 create mode 100644 foyer-storage/src/admission.rs
 create mode 100644 foyer-storage/src/device/error.rs
 create mode 100644 foyer-storage/src/device/fs.rs
 create mode 100644 foyer-storage/src/device/io_buffer.rs
 create mode 100644 foyer-storage/src/device/mod.rs
 create mode 100644 foyer-storage/src/error.rs
 create mode 100644 foyer-storage/src/flusher.rs
 create mode 100644 foyer-storage/src/indices.rs
 create mode 100644 foyer-storage/src/lib.rs
 create mode 100644 foyer-storage/src/reclaimer.rs
 create mode 100644 foyer-storage/src/region.rs
 create mode 100644 foyer-storage/src/region_manager.rs
 create mode 100644 foyer-storage/src/reinsertion.rs
 create mode 100644 foyer-storage/src/slice.rs
 create mode 100644 foyer-storage/src/store.rs
 create mode 100644 rustfmt.toml

diff --git a/.github/template/asan.yml b/.github/template/asan.yml
deleted file mode 100644
index e69de29b..00000000
diff --git a/.github/template/template.yml b/.github/template/template.yml
index dbf99535..b6482c4a 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -96,7 +96,16 @@ jobs:
           RUST_BACKTRACE: 1
           RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
           EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
-        run: |
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench --capacity 256
+      - name: Run foyer-storage-bench With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+        run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data --capacity 256
\ No newline at end of file
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 50abe598..0f8cc01a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -104,8 +104,17 @@ jobs:
           EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data --capacity 256
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench --capacity 256
+      - name: Run foyer-storage-bench With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
 
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index b1dca860..626cacbf 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -103,8 +103,17 @@ jobs:
           EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data --capacity 256
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench --capacity 256
+      - name: Run foyer-storage-bench With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
+          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
 concurrency:
   group: environment-${{ github.ref }}
   cancel-in-progress: true
diff --git a/Cargo.lock b/Cargo.lock
index 77800303..ed4a9c62 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -86,6 +86,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -135,7 +141,7 @@ checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990"
 dependencies = [
  "anstream",
  "anstyle",
- "bitflags",
+ "bitflags 1.3.2",
  "clap_lex",
  "strsim",
 ]
@@ -407,6 +413,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "tokio",
+ "tracing",
  "twox-hash",
 ]
 
@@ -439,6 +446,61 @@ dependencies = [
  "twox-hash",
 ]
 
+[[package]]
+name = "foyer-storage"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bitflags 2.3.3",
+ "bytes",
+ "bytesize",
+ "clap",
+ "cmsketch",
+ "crossbeam",
+ "foyer-common",
+ "foyer-intrusive",
+ "foyer-utils",
+ "futures",
+ "hdrhistogram",
+ "itertools",
+ "lazy_static",
+ "libc",
+ "memoffset 0.8.0",
+ "nix",
+ "parking_lot",
+ "paste",
+ "prometheus",
+ "rand",
+ "rand_mt",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "twox-hash",
+]
+
+[[package]]
+name = "foyer-storage-bench"
+version = "0.1.0"
+dependencies = [
+ "bytesize",
+ "clap",
+ "foyer-intrusive",
+ "foyer-storage",
+ "futures",
+ "hdrhistogram",
+ "itertools",
+ "libc",
+ "nix",
+ "parking_lot",
+ "rand",
+ "rand_mt",
+ "tempfile",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "foyer-utils"
 version = "0.1.0"
@@ -679,6 +741,15 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "matchers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
+dependencies = [
+ "regex-automata",
+]
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -736,7 +807,7 @@ version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "cfg-if",
  "libc",
  "memoffset 0.7.1",
@@ -754,6 +825,16 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "nu-ansi-term"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
+dependencies = [
+ "overload",
+ "winapi",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.15"
@@ -779,6 +860,12 @@ version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
 
+[[package]]
+name = "overload"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -910,7 +997,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -919,16 +1006,46 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "regex"
+version = "1.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
+dependencies = [
+ "regex-syntax 0.7.2",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+dependencies = [
+ "regex-syntax 0.6.29",
 ]
 
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
+[[package]]
+name = "regex-syntax"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
+
 [[package]]
 name = "rustix"
 version = "0.37.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "errno",
  "io-lifetimes",
  "libc",
@@ -942,6 +1059,15 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "sharded-slab"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
+dependencies = [
+ "lazy_static",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.1"
@@ -1022,6 +1148,16 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "thread_local"
+version = "1.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+]
+
 [[package]]
 name = "tokio"
 version = "1.28.1"
@@ -1079,6 +1215,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
 dependencies = [
  "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
+dependencies = [
+ "lazy_static",
+ "log",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
+dependencies = [
+ "matchers",
+ "nu-ansi-term",
+ "once_cell",
+ "regex",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
 ]
 
 [[package]]
@@ -1104,12 +1270,40 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
+[[package]]
+name = "valuable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-sys"
 version = "0.45.0"
diff --git a/Cargo.toml b/Cargo.toml
index 55ef2284..e7ad61ee 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,6 +6,8 @@ members = [
     "foyer-common",
     "foyer-intrusive",
     "foyer-policy",
+    "foyer-storage",
+    "foyer-storage-bench",
     "foyer-utils",
 ]
 
diff --git a/foyer-bench/src/analyze.rs b/foyer-bench/src/analyze.rs
index e07a0681..82a852cd 100644
--- a/foyer-bench/src/analyze.rs
+++ b/foyer-bench/src/analyze.rs
@@ -26,10 +26,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::{
+    path::Path,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
 
 use bytesize::ByteSize;
 use hdrhistogram::Histogram;
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 4cc8922d..d3c5d3d6 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -18,11 +18,15 @@ mod analyze;
 mod rate;
 mod utils;
 
-use std::fs::create_dir_all;
-use std::path::PathBuf;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::{
+    fs::create_dir_all,
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
 
 use analyze::{analyze, monitor, Metrics};
 use clap::Parser;
@@ -33,8 +37,7 @@ use foyer::{
 };
 use futures::future::join_all;
 use itertools::Itertools;
-use rand::rngs::StdRng;
-use rand::{Rng, SeedableRng};
+use rand::{rngs::StdRng, Rng, SeedableRng};
 
 use rate::RateLimiter;
 use tokio::sync::oneshot;
diff --git a/foyer-bench/src/utils.rs b/foyer-bench/src/utils.rs
index 620cd16e..d815da36 100644
--- a/foyer-bench/src/utils.rs
+++ b/foyer-bench/src/utils.rs
@@ -29,8 +29,7 @@
 use std::path::{Path, PathBuf};
 
 use itertools::Itertools;
-use nix::fcntl::readlink;
-use nix::sys::stat::stat;
+use nix::{fcntl::readlink, sys::stat::stat};
 
 #[allow(unused)]
 #[derive(PartialEq, Clone, Copy, Debug)]
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 61f53766..bfd1c5dc 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -16,8 +16,6 @@
 
 use bytes::{Buf, BufMut};
 
-pub trait Item = Sized + Send + Sync + 'static;
-
 pub trait Key:
     Sized
     + Send
@@ -28,6 +26,7 @@ pub trait Key:
     + PartialEq
     + Ord
     + PartialOrd
+    + Clone
     + std::fmt::Debug
 {
     fn serialized_len(&self) -> usize {
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 843d8461..9373da11 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -31,6 +31,7 @@ tokio = { version = "1", features = [
     "time",
     "signal",
 ] }
+tracing = "0.1"
 twox-hash = "1"
 
 [dev-dependencies]
diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index f000e062..4b160d10 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -14,8 +14,10 @@
 
 use std::ptr::NonNull;
 
-use crate::core::adapter::{Adapter, Link};
-use crate::core::pointer::PointerOps;
+use crate::core::{
+    adapter::{Adapter, Link},
+    pointer::PointerOps,
+};
 
 #[derive(Debug, Default)]
 pub struct DListLink {
@@ -52,6 +54,20 @@ where
     adapter: A,
 }
 
+impl<A> Drop for DList<A>
+where
+    A: Adapter<Link = DListLink>,
+{
+    fn drop(&mut self) {
+        let mut iter = self.iter_mut();
+        iter.front();
+        while iter.is_valid() {
+            iter.remove();
+        }
+        assert!(self.is_empty());
+    }
+}
+
 impl<A> DList<A>
 where
     A: Adapter<Link = DListLink>,
@@ -197,12 +213,12 @@ where
     }
 
     /// Move to head.
-    pub fn head(&mut self) {
+    pub fn front(&mut self) {
         self.link = self.dlist.head;
     }
 
     /// Move to head.
-    pub fn tail(&mut self) {
+    pub fn back(&mut self) {
         self.link = self.dlist.tail;
     }
 }
@@ -428,9 +444,11 @@ where
 
 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+
     use itertools::Itertools;
 
-    use crate::core::pointer::DefaultPointerOps;
+    use crate::{core::pointer::DefaultPointerOps, intrusive_adapter};
 
     use super::*;
 
@@ -480,6 +498,8 @@ mod tests {
         }
     }
 
+    intrusive_adapter! { DListArcAdapter = Arc<DListItem>: DListItem { link: DListLink } }
+
     #[test]
     fn test_dlist_simple() {
         let mut l = DList::<DListAdapter>::new();
@@ -510,4 +530,21 @@ mod tests {
         assert!(l.pop_front().is_none());
         assert_eq!(l.len(), 0);
     }
+
+    #[test]
+    fn test_arc_drop() {
+        let mut l = DList::<DListArcAdapter>::new();
+
+        let items = (0..10).map(|i| Arc::new(DListItem::new(i))).collect_vec();
+        for item in items.iter() {
+            l.push_back(item.clone());
+        }
+        for item in items.iter() {
+            assert_eq!(Arc::strong_count(item), 2);
+        }
+        drop(l);
+        for item in items.iter() {
+            assert_eq!(Arc::strong_count(item), 1);
+        }
+    }
 }
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index b759a8cd..10ab82f4 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -12,11 +12,13 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::fmt::Debug;
+
 use foyer_common::Key;
 
 use crate::core::pointer::PointerOps;
 
-pub trait Link: Send + Sync + 'static {
+pub trait Link: Send + Sync + 'static + Default + Debug {
     fn is_linked(&self) -> bool;
 }
 
@@ -87,23 +89,22 @@ pub unsafe trait KeyAdapter: Adapter {
 ///
 /// ```
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
-/// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter};
+/// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
 /// use foyer_intrusive::core::pointer::PointerOps;
 /// use foyer_intrusive::eviction::EvictionPolicy;
 /// use std::sync::Arc;
 ///
-/// pub struct Item<A, E>
+/// #[derive(Debug)]
+/// pub struct Item<L>
 /// where
-///     E: EvictionPolicy<A>,
-///     A: KeyAdapter<Link = E::Link>,
-///     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+///     L: Link
 /// {
-///     link: E::Link,
+///     link: L,
 ///     key: u64,
 /// }
 ///
-/// intrusive_adapter! { ItemAdapter<E> = Arc<Item<ItemAdapter<E>, E>>: Item<ItemAdapter<E>, E> { link: E::Link} where E:EvictionPolicy<ItemAdapter<E>> }
-/// key_adapter! { ItemAdapter<E> = Item<ItemAdapter<E>, E> { key: u64 } where E: EvictionPolicy<ItemAdapter<E>> }
+/// intrusive_adapter! { ItemAdapter<L> = Arc<Item<L>>: Item<L> { link: L} where L: Link }
+/// key_adapter! { ItemAdapter<L> = Item<L> { key: u64 } where L: Link }
 /// ```
 #[macro_export]
 macro_rules! intrusive_adapter {
@@ -112,6 +113,7 @@ macro_rules! intrusive_adapter {
     ) => {
         $vis struct $name<$($args),*> $($where_)* {
             pointer_ops: $crate::core::pointer::DefaultPointerOps<$pointer>,
+            _marker: std::marker::PhantomData<($($args),*)>
         }
 
         unsafe impl<$($args),*> Send for $name<$($args),*> $($where_)* {}
@@ -124,6 +126,7 @@ macro_rules! intrusive_adapter {
             fn new() -> Self {
                 Self {
                     pointer_ops: Default::default(),
+                    _marker: std::marker::PhantomData,
                 }
             }
 
@@ -157,7 +160,7 @@ macro_rules! intrusive_adapter {
         $vis:vis $name:ident<$($args:tt),*> = $($rest:tt)*
     ) => {
         intrusive_adapter! {@impl
-            $vis $name ($($args)*) = $($rest)*
+            $vis $name ($($args),*) = $($rest)*
         }
     };
 }
@@ -182,23 +185,22 @@ macro_rules! intrusive_adapter {
 ///
 /// ```
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
-/// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter};
+/// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
 /// use foyer_intrusive::core::pointer::PointerOps;
 /// use foyer_intrusive::eviction::EvictionPolicy;
 /// use std::sync::Arc;
 ///
-/// pub struct Item<A, E>
+/// #[derive(Debug)]
+/// pub struct Item<L>
 /// where
-///     E: EvictionPolicy<A>,
-///     A: KeyAdapter<Link = E::Link>,
-///     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+///     L: Link
 /// {
-///     link: E::Link,
+///     link: L,
 ///     key: u64,
 /// }
 ///
-/// intrusive_adapter! { ItemAdapter<E> = Arc<Item<ItemAdapter<E>, E>>: Item<ItemAdapter<E>, E> { link: E::Link} where E:EvictionPolicy<ItemAdapter<E>> }
-/// key_adapter! { ItemAdapter<E> = Item<ItemAdapter<E>, E> { key: u64 } where E: EvictionPolicy<ItemAdapter<E>> }
+/// intrusive_adapter! { ItemAdapter<L> = Arc<Item<L>>: Item<L> { link: L} where L: Link }
+/// key_adapter! { ItemAdapter<L> = Item<L> { key: u64 } where L: Link }
 /// ```
 #[macro_export]
 macro_rules! key_adapter {
@@ -227,11 +229,19 @@ macro_rules! key_adapter {
         $name:ident<$($args:tt),*> = $($rest:tt)*
     ) => {
         key_adapter! {@impl
-            $name ($($args)*) = $($rest)*
+            $name ($($args),*) = $($rest)*
         }
     };
 }
 
+macro_rules! t {
+    (
+        <$($args:tt),*>
+    ) => {};
+}
+
+t! { <A, B, C> }
+
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
@@ -242,6 +252,7 @@ mod tests {
 
     use super::*;
 
+    #[derive(Debug)]
     struct DListItem {
         link: DListLink,
         val: u64,
diff --git a/foyer-intrusive/src/core/pointer.rs b/foyer-intrusive/src/core/pointer.rs
index 027644e4..20daacef 100644
--- a/foyer-intrusive/src/core/pointer.rs
+++ b/foyer-intrusive/src/core/pointer.rs
@@ -26,18 +26,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::marker::PhantomData;
-use std::pin::Pin;
-use std::ptr::NonNull;
-use std::rc::Rc;
-use std::sync::Arc;
+use std::{fmt::Debug, marker::PhantomData, pin::Pin, ptr::NonNull, rc::Rc, sync::Arc};
 
 /// # Safety
 ///
 /// Pointer operations MUST be valid.
 pub unsafe trait PointerOps {
     type Item: ?Sized;
-    type Pointer;
+    type Pointer: Debug;
 
     /// # Safety
     ///
@@ -85,7 +81,7 @@ impl<Pointer> Default for DefaultPointerOps<Pointer> {
     }
 }
 
-unsafe impl<'a, T: ?Sized> PointerOps for DefaultPointerOps<&'a T> {
+unsafe impl<'a, T: ?Sized + Debug> PointerOps for DefaultPointerOps<&'a T> {
     type Item = T;
     type Pointer = &'a T;
 
@@ -105,7 +101,7 @@ unsafe impl<'a, T: ?Sized> PointerOps for DefaultPointerOps<&'a T> {
     }
 }
 
-unsafe impl<'a, T: ?Sized> PointerOps for DefaultPointerOps<Pin<&'a T>> {
+unsafe impl<'a, T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<&'a T>> {
     type Item = T;
     type Pointer = Pin<&'a T>;
 
@@ -125,7 +121,7 @@ unsafe impl<'a, T: ?Sized> PointerOps for DefaultPointerOps<Pin<&'a T>> {
     }
 }
 
-unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<NonNull<T>> {
+unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<NonNull<T>> {
     type Item = T;
     type Pointer = NonNull<T>;
 
@@ -143,7 +139,7 @@ unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<NonNull<T>> {
     }
 }
 
-unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Box<T>> {
+unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Box<T>> {
     type Item = T;
     type Pointer = Box<T>;
 
@@ -163,7 +159,7 @@ unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Box<T>> {
     }
 }
 
-unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Box<T>>> {
+unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<Box<T>>> {
     type Item = T;
     type Pointer = Pin<Box<T>>;
 
@@ -183,7 +179,7 @@ unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Box<T>>> {
     }
 }
 
-unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Rc<T>> {
+unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Rc<T>> {
     type Item = T;
     type Pointer = Rc<T>;
 
@@ -203,7 +199,7 @@ unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Rc<T>> {
     }
 }
 
-unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Rc<T>>> {
+unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<Rc<T>>> {
     type Item = T;
     type Pointer = Pin<Rc<T>>;
 
@@ -223,7 +219,7 @@ unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Rc<T>>> {
     }
 }
 
-unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Arc<T>> {
+unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Arc<T>> {
     type Item = T;
     type Pointer = Arc<T>;
 
@@ -243,7 +239,7 @@ unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Arc<T>> {
     }
 }
 
-unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Arc<T>>> {
+unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<Arc<T>>> {
     type Item = T;
     type Pointer = Pin<Arc<T>>;
 
@@ -263,7 +259,7 @@ unsafe impl<T: ?Sized> PointerOps for DefaultPointerOps<Pin<Arc<T>>> {
     }
 }
 
-unsafe impl<T: ?Sized> DowngradablePointerOps for DefaultPointerOps<Rc<T>> {
+unsafe impl<T: ?Sized + Debug> DowngradablePointerOps for DefaultPointerOps<Rc<T>> {
     type WeakPointer = std::rc::Weak<T>;
 
     fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer {
@@ -271,7 +267,7 @@ unsafe impl<T: ?Sized> DowngradablePointerOps for DefaultPointerOps<Rc<T>> {
     }
 }
 
-unsafe impl<T: ?Sized> DowngradablePointerOps for DefaultPointerOps<Arc<T>> {
+unsafe impl<T: ?Sized + Debug> DowngradablePointerOps for DefaultPointerOps<Arc<T>> {
     type WeakPointer = std::sync::Weak<T>;
 
     fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer {
@@ -282,12 +278,7 @@ unsafe impl<T: ?Sized> DowngradablePointerOps for DefaultPointerOps<Arc<T>> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::boxed::Box;
-    use std::fmt::Debug;
-    use std::mem;
-    use std::pin::Pin;
-    use std::rc::Rc;
-    use std::sync::Arc;
+    use std::{boxed::Box, fmt::Debug, mem, pin::Pin, rc::Rc, sync::Arc};
 
     /// Clones a `PointerOps::Pointer` from a `*const PointerOps::Value`
     ///
@@ -301,8 +292,7 @@ mod tests {
     where
         T::Pointer: Clone,
     {
-        use std::mem::ManuallyDrop;
-        use std::ops::Deref;
+        use std::{mem::ManuallyDrop, ops::Deref};
 
         /// Guard which converts an pointer back into its raw version
         /// when it gets dropped. This makes sure we also perform a full
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 324d650e..9d5870fb 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -26,20 +26,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::collections::dlist::DList;
-use crate::collections::dlist::DListIter;
-use crate::core::adapter::Adapter;
-use crate::core::adapter::KeyAdapter;
-use crate::core::adapter::Link;
-use crate::core::pointer::PointerOps;
-use crate::intrusive_adapter;
+use crate::{
+    collections::dlist::{DList, DListIter},
+    core::{
+        adapter::{Adapter, KeyAdapter, Link},
+        pointer::PointerOps,
+    },
+    intrusive_adapter,
+};
 
-use std::mem::ManuallyDrop;
-use std::ptr::NonNull;
+use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use cmsketch::CMSketchUsize;
-use std::hash::Hash;
-use std::hash::Hasher;
+use std::hash::{Hash, Hasher};
 use twox_hash::XxHash64;
 
 use crate::collections::dlist::DListLink;
@@ -155,10 +154,25 @@ where
     adapter: A,
 }
 
-impl<A> Lfu<A>
+impl<A> Drop for Lfu<A>
 where
     A: KeyAdapter<Link = LfuLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    fn drop(&mut self) {
+        let mut to_remove = vec![];
+        for ptr in self.iter() {
+            to_remove.push(ptr.clone());
+        }
+        for ptr in to_remove {
+            self.remove(&ptr);
+        }
+    }
+}
 
+impl<A> Lfu<A>
+where
+    A: KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     pub fn new(config: LfuConfig) -> Self {
@@ -242,8 +256,8 @@ where
         let mut iter_main = self.lru_main.iter();
         let mut iter_tiny = self.lru_tiny.iter();
 
-        iter_main.tail();
-        iter_tiny.tail();
+        iter_main.back();
+        iter_tiny.back();
 
         LfuIter {
             lfu: self,
@@ -519,6 +533,7 @@ where
     }
 
     fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        tracing::debug!("[lfu] insert {:?}", ptr);
         self.insert(ptr)
     }
 
@@ -526,10 +541,12 @@ where
         &mut self,
         ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
     ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
+        tracing::debug!("[lfu] remove {:?}", ptr);
         self.remove(ptr)
     }
 
     fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        tracing::debug!("[lfu] access {:?}", ptr);
         self.access(ptr)
     }
 
@@ -548,6 +565,7 @@ mod tests {
 
     use super::*;
 
+    #[derive(Debug)]
     struct LfuItem {
         link: LfuLink,
         key: u64,
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 86584da5..db8f9ac1 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -85,6 +85,22 @@ where
     adapter: A,
 }
 
+impl<A> Drop for Lru<A>
+where
+    A: KeyAdapter<Link = LruLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    fn drop(&mut self) {
+        let mut to_remove = vec![];
+        for ptr in self.iter() {
+            to_remove.push(ptr.clone());
+        }
+        for ptr in to_remove {
+            self.remove(&ptr);
+        }
+    }
+}
+
 impl<A> Lru<A>
 where
     A: KeyAdapter<Link = LruLink>,
@@ -162,7 +178,7 @@ where
 
     fn iter(&self) -> LruIter<'_, A> {
         let mut iter = self.lru.iter();
-        iter.tail();
+        iter.back();
         LruIter {
             iter,
             lru: self,
@@ -363,6 +379,7 @@ where
     }
 
     fn insert(&mut self, ptr: <<A>::PointerOps as PointerOps>::Pointer) {
+        tracing::debug!("[lru] insert {:?}", ptr);
         self.insert(ptr)
     }
 
@@ -370,10 +387,12 @@ where
         &mut self,
         ptr: &<<A>::PointerOps as PointerOps>::Pointer,
     ) -> <<A>::PointerOps as PointerOps>::Pointer {
+        tracing::debug!("[lru] remove {:?}", ptr);
         self.remove(ptr)
     }
 
     fn access(&mut self, ptr: &<<A>::PointerOps as PointerOps>::Pointer) {
+        tracing::debug!("[lru] access {:?}", ptr);
         self.access(ptr)
     }
 
@@ -392,6 +411,7 @@ mod tests {
 
     use super::*;
 
+    #[derive(Debug)]
     struct LruItem {
         link: LruLink,
         key: u64,
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index 9dbd3f3a..bd1c8b07 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -12,10 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::core::adapter::{Adapter, KeyAdapter, Link};
-use crate::core::pointer::PointerOps;
+use crate::core::{
+    adapter::{Adapter, KeyAdapter, Link},
+    pointer::PointerOps,
+};
 
-pub trait Config = Send + Sync + 'static;
+use std::fmt::Debug;
+
+pub trait Config = Send + Sync + 'static + Debug;
 
 pub trait EvictionPolicy<A>: Send + Sync + 'static
 where
diff --git a/foyer-policy/src/eviction/lru.rs b/foyer-policy/src/eviction/lru.rs
index 8e542311..2c3f33f6 100644
--- a/foyer-policy/src/eviction/lru.rs
+++ b/foyer-policy/src/eviction/lru.rs
@@ -26,11 +26,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::ptr::NonNull;
-use std::time::SystemTime;
+use std::{ptr::NonNull, time::SystemTime};
 
-use foyer_utils::dlist::{DList, Entry, Iter};
-use foyer_utils::intrusive_dlist;
+use foyer_utils::{
+    dlist::{DList, Entry, Iter},
+    intrusive_dlist,
+};
 
 use super::Item;
 
diff --git a/foyer-policy/src/eviction/tinylfu.rs b/foyer-policy/src/eviction/tinylfu.rs
index 5f75fd37..5b87e0ea 100644
--- a/foyer-policy/src/eviction/tinylfu.rs
+++ b/foyer-policy/src/eviction/tinylfu.rs
@@ -26,15 +26,15 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::hash::Hasher;
-use std::ptr::NonNull;
-use std::time::SystemTime;
+use std::{hash::Hasher, ptr::NonNull, time::SystemTime};
 
 use cmsketch::CMSketchUsize;
 use twox_hash::XxHash64;
 
-use foyer_utils::dlist::{DList, Entry, Iter};
-use foyer_utils::intrusive_dlist;
+use foyer_utils::{
+    dlist::{DList, Entry, Iter},
+    intrusive_dlist,
+};
 
 use super::Item;
 
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
new file mode 100644
index 00000000..3254b026
--- /dev/null
+++ b/foyer-storage-bench/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+name = "foyer-storage-bench"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+foyer-intrusive = { path = "../foyer-intrusive" }
+foyer-storage = { path = "../foyer-storage" }
+futures = "0.3"
+hdrhistogram = "7"
+itertools = "0.10.5"
+libc = "0.2"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+rand = "0.8.5"
+rand_mt = "4.2.1"
+tempfile = "3"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
new file mode 100644
index 00000000..82a852cd
--- /dev/null
+++ b/foyer-storage-bench/src/analyze.rs
@@ -0,0 +1,278 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    path::Path,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
+
+use bytesize::ByteSize;
+use hdrhistogram::Histogram;
+use parking_lot::RwLock;
+use tokio::sync::oneshot;
+
+use crate::utils::{iostat, IoStat};
+
+const SECTOR_SIZE: usize = 512;
+
+// latencies are measured by 'us'
+#[derive(Clone, Copy, Debug)]
+pub struct Analysis {
+    disk_read_iops: f64,
+    disk_read_throughput: f64,
+    disk_write_iops: f64,
+    disk_write_throughput: f64,
+
+    insert_iops: f64,
+    insert_throughput: f64,
+    insert_lat_p50: u64,
+    insert_lat_p90: u64,
+    insert_lat_p99: u64,
+
+    get_iops: f64,
+    get_miss: f64,
+    get_miss_lat_p50: u64,
+    get_miss_lat_p90: u64,
+    get_miss_lat_p99: u64,
+    get_hit_lat_p50: u64,
+    get_hit_lat_p90: u64,
+    get_hit_lat_p99: u64,
+}
+
+#[derive(Default, Clone, Copy, Debug)]
+pub struct MetricsDump {
+    pub insert_ios: usize,
+    pub insert_bytes: usize,
+    pub insert_lat_p50: u64,
+    pub insert_lat_p90: u64,
+    pub insert_lat_p99: u64,
+
+    pub get_ios: usize,
+    pub get_miss_ios: usize,
+    pub get_hit_lat_p50: u64,
+    pub get_hit_lat_p90: u64,
+    pub get_hit_lat_p99: u64,
+    pub get_miss_lat_p50: u64,
+    pub get_miss_lat_p90: u64,
+    pub get_miss_lat_p99: u64,
+}
+
+#[derive(Clone, Debug)]
+pub struct Metrics {
+    pub insert_ios: Arc<AtomicUsize>,
+    pub insert_bytes: Arc<AtomicUsize>,
+    pub insert_lats: Arc<RwLock<Histogram<u64>>>,
+
+    pub get_ios: Arc<AtomicUsize>,
+    pub get_miss_ios: Arc<AtomicUsize>,
+    pub get_hit_lats: Arc<RwLock<Histogram<u64>>>,
+    pub get_miss_lats: Arc<RwLock<Histogram<u64>>>,
+}
+
+impl Default for Metrics {
+    fn default() -> Self {
+        Self {
+            insert_ios: Arc::new(AtomicUsize::new(0)),
+            insert_bytes: Arc::new(AtomicUsize::new(0)),
+            insert_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+
+            get_ios: Arc::new(AtomicUsize::new(0)),
+            get_miss_ios: Arc::new(AtomicUsize::new(0)),
+            get_hit_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+            get_miss_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+        }
+    }
+}
+
+impl Metrics {
+    pub fn dump(&self) -> MetricsDump {
+        let insert_lats = self.insert_lats.read();
+        let get_hit_lats = self.get_hit_lats.read();
+        let get_miss_lats = self.get_miss_lats.read();
+        MetricsDump {
+            insert_ios: self.insert_ios.load(Ordering::Relaxed),
+            insert_bytes: self.insert_bytes.load(Ordering::Relaxed),
+            insert_lat_p50: insert_lats.value_at_quantile(0.5),
+            insert_lat_p90: insert_lats.value_at_quantile(0.9),
+            insert_lat_p99: insert_lats.value_at_quantile(0.99),
+
+            get_ios: self.get_ios.load(Ordering::Relaxed),
+            get_miss_ios: self.get_miss_ios.load(Ordering::Relaxed),
+            get_hit_lat_p50: get_hit_lats.value_at_quantile(0.5),
+            get_hit_lat_p90: get_hit_lats.value_at_quantile(0.9),
+            get_hit_lat_p99: get_hit_lats.value_at_quantile(0.99),
+            get_miss_lat_p50: get_miss_lats.value_at_quantile(0.5),
+            get_miss_lat_p90: get_miss_lats.value_at_quantile(0.9),
+            get_miss_lat_p99: get_miss_lats.value_at_quantile(0.99),
+        }
+    }
+}
+
+impl std::fmt::Display for Analysis {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let disk_read_throughput = ByteSize::b(self.disk_read_throughput as u64);
+        let disk_write_throughput = ByteSize::b(self.disk_write_throughput as u64);
+        let disk_total_throughput = disk_read_throughput + disk_write_throughput;
+
+        // disk statics
+        writeln!(
+            f,
+            "disk total iops: {:.1}",
+            self.disk_write_iops + self.disk_read_iops
+        )?;
+        writeln!(
+            f,
+            "disk total throughput: {}/s",
+            disk_total_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "disk read iops: {:.1}", self.disk_read_iops)?;
+        writeln!(
+            f,
+            "disk read throughput: {}/s",
+            disk_read_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "disk write iops: {:.1}", self.disk_write_iops)?;
+        writeln!(
+            f,
+            "disk write throughput: {}/s",
+            disk_write_throughput.to_string_as(true)
+        )?;
+
+        // insert statics
+        let insert_throughput = ByteSize::b(self.insert_throughput as u64);
+        writeln!(f, "insert iops: {:.1}/s", self.insert_iops)?;
+        writeln!(
+            f,
+            "insert throughput: {}/s",
+            insert_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
+        writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
+        writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
+
+        // get statics
+        writeln!(f, "get iops: {:.1}/s", self.get_iops)?;
+        writeln!(f, "get miss: {:.2}% ", self.get_miss * 100f64)?;
+        writeln!(f, "get hit lat p50: {}us", self.get_hit_lat_p50)?;
+        writeln!(f, "get hit lat p90: {}us", self.get_hit_lat_p90)?;
+        writeln!(f, "get hit lat p99: {}us", self.get_hit_lat_p99)?;
+        writeln!(f, "get miss lat p50: {}us", self.get_miss_lat_p50)?;
+        writeln!(f, "get miss lat p90: {}us", self.get_miss_lat_p90)?;
+        writeln!(f, "get miss lat p99: {}us", self.get_miss_lat_p99)?;
+
+        Ok(())
+    }
+}
+
+pub fn analyze(
+    duration: Duration,
+    iostat_start: &IoStat,
+    iostat_end: &IoStat,
+    metrics_dump_start: &MetricsDump,
+    metrics_dump_end: &MetricsDump,
+) -> Analysis {
+    let secs = duration.as_secs_f64();
+    let disk_read_iops = (iostat_end.read_ios - iostat_start.read_ios) as f64 / secs;
+    let disk_read_throughput =
+        (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+    let disk_write_iops = (iostat_end.write_ios - iostat_start.write_ios) as f64 / secs;
+    let disk_write_throughput =
+        (iostat_end.write_sectors - iostat_start.write_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+
+    let insert_iops = (metrics_dump_end.insert_ios - metrics_dump_start.insert_ios) as f64 / secs;
+    let insert_throughput =
+        (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
+
+    let get_iops = (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64 / secs;
+    let get_miss = (metrics_dump_end.get_miss_ios - metrics_dump_start.get_miss_ios) as f64
+        / (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64;
+
+    Analysis {
+        disk_read_iops,
+        disk_read_throughput,
+        disk_write_iops,
+        disk_write_throughput,
+
+        insert_iops,
+        insert_throughput,
+        insert_lat_p50: metrics_dump_end.insert_lat_p50,
+        insert_lat_p90: metrics_dump_end.insert_lat_p90,
+        insert_lat_p99: metrics_dump_end.insert_lat_p99,
+
+        get_iops,
+        get_miss,
+        get_hit_lat_p50: metrics_dump_end.get_hit_lat_p50,
+        get_hit_lat_p90: metrics_dump_end.get_hit_lat_p90,
+        get_hit_lat_p99: metrics_dump_end.get_hit_lat_p99,
+        get_miss_lat_p50: metrics_dump_end.get_miss_lat_p50,
+        get_miss_lat_p90: metrics_dump_end.get_miss_lat_p90,
+        get_miss_lat_p99: metrics_dump_end.get_miss_lat_p99,
+    }
+}
+
+pub async fn monitor(
+    iostat_path: impl AsRef<Path>,
+    interval: Duration,
+    metrics: Metrics,
+    mut stop: oneshot::Receiver<()>,
+) {
+    let mut stat = iostat(&iostat_path);
+    let mut metrics_dump = metrics.dump();
+    loop {
+        let start = Instant::now();
+        match stop.try_recv() {
+            Err(oneshot::error::TryRecvError::Empty) => {}
+            _ => return,
+        }
+
+        tokio::time::sleep(interval).await;
+        let new_stat = iostat(&iostat_path);
+        let new_metrics_dump = metrics.dump();
+        let analysis = analyze(
+            // interval may have ~ +7% error
+            start.elapsed(),
+            &stat,
+            &new_stat,
+            &metrics_dump,
+            &new_metrics_dump,
+        );
+        println!("{}", analysis);
+        stat = new_stat;
+        metrics_dump = new_metrics_dump;
+    }
+}
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
new file mode 100644
index 00000000..d2689989
--- /dev/null
+++ b/foyer-storage-bench/src/main.rs
@@ -0,0 +1,386 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(let_chains)]
+
+mod analyze;
+mod rate;
+mod utils;
+
+use std::{
+    fs::create_dir_all,
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
+
+use analyze::{analyze, monitor, Metrics};
+use clap::Parser;
+
+use foyer_intrusive::eviction::lfu::{Lfu, LfuConfig, LfuLink};
+use foyer_storage::{
+    admission::AdmitAll,
+    device::{
+        fs::{FsDevice, FsDeviceConfig},
+        io_buffer::AlignedAllocator,
+    },
+    region_manager::RegionEpItemAdapter,
+    reinsertion::ReinsertNone,
+    store::{Store, StoreConfig},
+};
+use futures::future::join_all;
+use itertools::Itertools;
+use rand::{rngs::StdRng, Rng, SeedableRng};
+
+use rate::RateLimiter;
+use tokio::sync::oneshot;
+use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt};
+use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
+
+#[derive(Parser, Debug, Clone)]
+pub struct Args {
+    /// dir for cache data
+    #[arg(short, long)]
+    dir: String,
+
+    /// (MiB)
+    #[arg(long, default_value_t = 1024)]
+    capacity: usize,
+
+    /// (s)
+    #[arg(short, long, default_value_t = 60)]
+    time: u64,
+
+    #[arg(long, default_value_t = 16)]
+    concurrency: usize,
+
+    /// must be power of 2
+    #[arg(long, default_value_t = 16)]
+    pools: usize,
+
+    /// (s)
+    #[arg(long, default_value_t = 2)]
+    report_interval: u64,
+
+    /// Some filesystem (e.g. btrfs) can span across multiple block devices and it's hard to decide
+    /// which device to moitor. Use this argument to specify which block device to monitor.
+    #[arg(long, default_value = "")]
+    iostat_dev: String,
+
+    #[arg(long, default_value_t = 0.0)]
+    w_rate: f64,
+
+    #[arg(long, default_value_t = 0.0)]
+    r_rate: f64,
+
+    #[arg(long, default_value_t = 64 * 1024)]
+    entry_size: usize,
+
+    #[arg(long, default_value_t = 10000)]
+    lookup_range: u64,
+
+    /// (MiB)
+    #[arg(long, default_value_t = 64)]
+    region_size: usize,
+
+    /// (MiB)
+    #[arg(long, default_value_t = 1024)]
+    buffer_pool_size: usize,
+
+    #[arg(long, default_value_t = 4)]
+    flushers: usize,
+
+    #[arg(long, default_value_t = 4)]
+    reclaimers: usize,
+
+    #[arg(long, default_value_t = 4096)]
+    align: usize,
+
+    #[arg(long, default_value_t = 16 * 1024)]
+    io_size: usize,
+}
+
+impl Args {
+    fn verify(&self) {
+        assert!(self.pools.is_power_of_two());
+    }
+}
+
+type TStore = Store<
+    u64,
+    Vec<u8>,
+    AlignedAllocator,
+    FsDevice,
+    Lfu<RegionEpItemAdapter<LfuLink>>,
+    AdmitAll<u64, Vec<u8>>,
+    ReinsertNone<u64, Vec<u8>>,
+    LfuLink,
+>;
+
+fn is_send_sync_static<T: Send + Sync + 'static>() {}
+
+#[tokio::main]
+async fn main() {
+    is_send_sync_static::<TStore>();
+
+    tracing_subscriber::registry()
+        .with(tracing_subscriber::fmt::layer())
+        .with(tracing_subscriber::EnvFilter::from_default_env())
+        .init();
+
+    let args = Args::parse();
+    args.verify();
+
+    println!("{:#?}", args);
+
+    create_dir_all(&args.dir).unwrap();
+
+    let iostat_path = match detect_fs_type(&args.dir) {
+        FsType::Tmpfs => panic!("tmpfs is not supported with benches"),
+        FsType::Btrfs => {
+            if args.iostat_dev.is_empty() {
+                panic!("cannot decide which block device to monitor for btrfs, please specify device name with \'--iostat-dev\'")
+            } else {
+                dev_stat_path(&args.iostat_dev)
+            }
+        }
+        _ => file_stat_path(&args.dir),
+    };
+
+    let metrics = Metrics::default();
+
+    let iostat_start = iostat(&iostat_path);
+    let metrics_dump_start = metrics.dump();
+    let time = Instant::now();
+
+    let eviction_config = LfuConfig {
+        window_to_cache_size_ratio: 1,
+        tiny_lru_capacity_ratio: 0.01,
+    };
+
+    let device_config = FsDeviceConfig {
+        dir: PathBuf::from(&args.dir),
+        capacity: args.capacity * 1024 * 1024,
+        file_capacity: args.region_size * 1024 * 1024,
+        align: args.align,
+        io_size: args.io_size,
+    };
+
+    let config = StoreConfig {
+        eviction_config,
+        device_config,
+        admission: AdmitAll::default(),
+        reinsertion: ReinsertNone::default(),
+        buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
+        flushers: args.flushers,
+        reclaimers: args.reclaimers,
+    };
+
+    println!("{:#?}", config);
+
+    let store = TStore::open(config).await.unwrap();
+
+    let (iostat_stop_tx, iostat_stop_rx) = oneshot::channel();
+    let (bench_stop_txs, bench_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
+        (0..args.concurrency).map(|_| oneshot::channel()).unzip();
+
+    let handle_monitor = tokio::spawn({
+        let iostat_path = iostat_path.clone();
+        let metrics = metrics.clone();
+
+        monitor(
+            iostat_path,
+            Duration::from_secs(args.report_interval),
+            metrics,
+            iostat_stop_rx,
+        )
+    });
+    let handle_signal = tokio::spawn(async move {
+        tokio::signal::ctrl_c().await.unwrap();
+        for bench_stop_tx in bench_stop_txs {
+            bench_stop_tx.send(()).unwrap();
+        }
+        iostat_stop_tx.send(()).unwrap();
+    });
+    let handles = (bench_stop_rxs)
+        .into_iter()
+        .map(|stop| tokio::spawn(bench(args.clone(), store.clone(), metrics.clone(), stop)))
+        .collect_vec();
+
+    for handle in handles {
+        handle.await.unwrap();
+    }
+    handle_monitor.abort();
+    handle_signal.abort();
+
+    let iostat_end = iostat(&iostat_path);
+    let metrics_dump_end = metrics.dump();
+    let analysis = analyze(
+        time.elapsed(),
+        &iostat_start,
+        &iostat_end,
+        &metrics_dump_start,
+        &metrics_dump_end,
+    );
+    println!("\nTotal:\n{}", analysis);
+}
+
+async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::Receiver<()>) {
+    let w_rate = if args.w_rate == 0.0 {
+        None
+    } else {
+        Some(args.w_rate)
+    };
+    let r_rate = if args.r_rate == 0.0 {
+        None
+    } else {
+        Some(args.r_rate)
+    };
+
+    let index = Arc::new(AtomicU64::new(0));
+
+    let (w_stop_tx, w_stop_rx) = oneshot::channel();
+    let (r_stop_tx, r_stop_rx) = oneshot::channel();
+
+    let handle_w = tokio::spawn(write(
+        args.entry_size,
+        w_rate,
+        index.clone(),
+        store.clone(),
+        args.time,
+        metrics.clone(),
+        w_stop_rx,
+    ));
+    let handle_r = tokio::spawn(read(
+        args.entry_size,
+        r_rate,
+        index.clone(),
+        store.clone(),
+        args.time,
+        metrics.clone(),
+        r_stop_rx,
+        args.lookup_range,
+    ));
+    tokio::spawn(async move {
+        if let Ok(()) = stop.await {
+            let _ = w_stop_tx.send(());
+            let _ = r_stop_tx.send(());
+        }
+    });
+
+    join_all([handle_r, handle_w]).await;
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn write(
+    entry_size: usize,
+    rate: Option<f64>,
+    index: Arc<AtomicU64>,
+    store: Arc<TStore>,
+    time: u64,
+    metrics: Metrics,
+    mut stop: oneshot::Receiver<()>,
+) {
+    let start = Instant::now();
+
+    let mut limiter = rate.map(RateLimiter::new);
+
+    loop {
+        match stop.try_recv() {
+            Err(oneshot::error::TryRecvError::Empty) => {}
+            _ => return,
+        }
+        if start.elapsed().as_secs() >= time {
+            return;
+        }
+
+        let idx = index.fetch_add(1, Ordering::Relaxed);
+        // TODO(MrCroxx): Use random content?
+        let data = vec![b'x'; entry_size];
+        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
+            tokio::time::sleep(wait).await;
+        }
+
+        let time = Instant::now();
+        store.insert(idx, data).await.unwrap();
+        metrics
+            .insert_lats
+            .write()
+            .record(time.elapsed().as_micros() as u64)
+            .expect("record out of range");
+        metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
+        metrics
+            .insert_bytes
+            .fetch_add(entry_size, Ordering::Relaxed);
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn read(
+    entry_size: usize,
+    rate: Option<f64>,
+    index: Arc<AtomicU64>,
+    store: Arc<TStore>,
+    time: u64,
+    metrics: Metrics,
+    mut stop: oneshot::Receiver<()>,
+    look_up_range: u64,
+) {
+    let start = Instant::now();
+
+    let mut limiter = rate.map(RateLimiter::new);
+
+    let mut rng = StdRng::seed_from_u64(0);
+
+    loop {
+        match stop.try_recv() {
+            Err(oneshot::error::TryRecvError::Empty) => {}
+            _ => return,
+        }
+        if start.elapsed().as_secs() >= time {
+            return;
+        }
+
+        let idx_max = index.load(Ordering::Relaxed);
+        let idx = rng.gen_range(std::cmp::max(idx_max, look_up_range) - look_up_range..=idx_max);
+
+        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
+            tokio::time::sleep(wait).await;
+        }
+
+        let time = Instant::now();
+        let hit = store.lookup(&idx).await.unwrap().is_some();
+        let lat = time.elapsed().as_micros() as u64;
+        if hit {
+            metrics
+                .get_hit_lats
+                .write()
+                .record(lat)
+                .expect("record out of range");
+        } else {
+            metrics
+                .get_miss_lats
+                .write()
+                .record(lat)
+                .expect("record out of range");
+            metrics.get_miss_ios.fetch_add(1, Ordering::Relaxed);
+        }
+        metrics.get_ios.fetch_add(1, Ordering::Relaxed);
+
+        tokio::task::consume_budget().await;
+    }
+}
diff --git a/foyer-storage-bench/src/rate.rs b/foyer-storage-bench/src/rate.rs
new file mode 100644
index 00000000..82164aab
--- /dev/null
+++ b/foyer-storage-bench/src/rate.rs
@@ -0,0 +1,59 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::{Duration, Instant};
+
+pub struct RateLimiter {
+    capacity: f64,
+    quota: f64,
+
+    last: Instant,
+}
+
+impl RateLimiter {
+    pub fn new(capacity: f64) -> Self {
+        Self {
+            capacity,
+            quota: 0.0,
+            last: Instant::now(),
+        }
+    }
+
+    pub fn consume(&mut self, weight: f64) -> Option<Duration> {
+        let now = Instant::now();
+        let refill = now.duration_since(self.last).as_secs_f64() * self.capacity;
+        self.last = now;
+        self.quota = f64::min(self.quota + refill, self.capacity);
+        self.quota -= weight;
+        if self.quota >= 0.0 {
+            return None;
+        }
+        let wait = Duration::from_secs_f64((-self.quota) / self.capacity);
+        Some(wait)
+    }
+}
diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
new file mode 100644
index 00000000..d815da36
--- /dev/null
+++ b/foyer-storage-bench/src/utils.rs
@@ -0,0 +1,170 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::{Path, PathBuf};
+
+use itertools::Itertools;
+use nix::{fcntl::readlink, sys::stat::stat};
+
+#[allow(unused)]
+#[derive(PartialEq, Clone, Copy, Debug)]
+pub enum FsType {
+    Xfs,
+    Ext4,
+    Btrfs,
+    Tmpfs,
+    Others,
+}
+
+#[allow(unused)]
+pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
+    #[cfg(target_os = "linux")]
+    {
+        use nix::sys::statfs::{
+            statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC,
+        };
+        let fs_stat = statfs(path.as_ref()).unwrap();
+        match fs_stat.filesystem_type() {
+            XFS_SUPER_MAGIC => FsType::Xfs,
+            EXT4_SUPER_MAGIC => FsType::Ext4,
+            BTRFS_SUPER_MAGIC => FsType::Btrfs,
+            TMPFS_MAGIC => FsType::Tmpfs,
+            _ => FsType::Others,
+        }
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    FsType::Others
+}
+
+/// Given a normal file path, returns the containing block device static file path (of the
+/// partition).
+pub fn file_stat_path(path: impl AsRef<Path>) -> PathBuf {
+    let st_dev = stat(path.as_ref()).unwrap().st_dev;
+
+    let major = unsafe { libc::major(st_dev) };
+    let minor = unsafe { libc::minor(st_dev) };
+
+    let dev = PathBuf::from("/dev/block").join(format!("{}:{}", major, minor));
+
+    let linkname = readlink(&dev).unwrap();
+    let devname = Path::new(linkname.as_os_str()).file_name().unwrap();
+    dev_stat_path(devname.to_str().unwrap())
+}
+
+pub fn dev_stat_path(devname: &str) -> PathBuf {
+    let classpath = Path::new("/sys/class/block").join(devname);
+    let devclass = readlink(&classpath).unwrap();
+
+    let devpath = Path::new(&devclass);
+    Path::new("/sys")
+        .join(devpath.strip_prefix("../..").unwrap())
+        .join("stat")
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct IoStat {
+    /// read I/Os       requests      number of read I/Os processed
+    pub read_ios: usize,
+    /// read merges     requests      number of read I/Os merged with in-queue I/O
+    pub read_merges: usize,
+    /// read sectors    sectors       number of sectors read
+    pub read_sectors: usize,
+    /// read ticks      milliseconds  total wait time for read requests
+    pub read_ticks: usize,
+    /// write I/Os      requests      number of write I/Os processed
+    pub write_ios: usize,
+    /// write merges    requests      number of write I/Os merged with in-queue I/O
+    pub write_merges: usize,
+    /// write sectors   sectors       number of sectors written
+    pub write_sectors: usize,
+    /// write ticks     milliseconds  total wait time for write requests
+    pub write_ticks: usize,
+    /// in_flight       requests      number of I/Os currently in flight
+    pub in_flight: usize,
+    /// io_ticks        milliseconds  total time this block device has been active
+    pub io_ticks: usize,
+    /// time_in_queue   milliseconds  total wait time for all requests
+    pub time_in_queue: usize,
+    /// discard I/Os    requests      number of discard I/Os processed
+    pub discard_ios: usize,
+    /// discard merges  requests      number of discard I/Os merged with in-queue I/O
+    pub discard_merges: usize,
+    /// discard sectors sectors       number of sectors discarded
+    pub discard_sectors: usize,
+    /// discard ticks   milliseconds  total wait time for discard requests
+    pub discard_ticks: usize,
+    /// flush I/Os      requests      number of flush I/Os processed
+    pub flush_ios: usize,
+    /// flush ticks     milliseconds  total wait time for flush requests
+    pub flush_ticks: usize,
+}
+
+/// Given the device static file path and get the iostat.
+pub fn iostat(path: impl AsRef<Path>) -> IoStat {
+    let content = std::fs::read_to_string(path.as_ref()).unwrap();
+    let nums = content.split_ascii_whitespace().collect_vec();
+
+    let read_ios = nums[0].parse().unwrap();
+    let read_merges = nums[1].parse().unwrap();
+    let read_sectors = nums[2].parse().unwrap();
+    let read_ticks = nums[3].parse().unwrap();
+    let write_ios = nums[4].parse().unwrap();
+    let write_merges = nums[5].parse().unwrap();
+    let write_sectors = nums[6].parse().unwrap();
+    let write_ticks = nums[7].parse().unwrap();
+    let in_flight = nums[8].parse().unwrap();
+    let io_ticks = nums[9].parse().unwrap();
+    let time_in_queue = nums[10].parse().unwrap();
+    let discard_ios = nums[11].parse().unwrap();
+    let discard_merges = nums[12].parse().unwrap();
+    let discard_sectors = nums[13].parse().unwrap();
+    let discard_ticks = nums[14].parse().unwrap();
+    let flush_ios = nums[15].parse().unwrap();
+    let flush_ticks = nums[16].parse().unwrap();
+
+    IoStat {
+        read_ios,
+        read_merges,
+        read_sectors,
+        read_ticks,
+        write_ios,
+        write_merges,
+        write_sectors,
+        write_ticks,
+        in_flight,
+        io_ticks,
+        time_in_queue,
+        discard_ios,
+        discard_merges,
+        discard_sectors,
+        discard_ticks,
+        flush_ios,
+        flush_ticks,
+    }
+}
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
new file mode 100644
index 00000000..0a4093ca
--- /dev/null
+++ b/foyer-storage/Cargo.toml
@@ -0,0 +1,46 @@
+[package]
+name = "foyer-storage"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1"
+bitflags = "2.3.1"
+bytes = "1"
+cmsketch = "0.1"
+crossbeam = "0.8"
+foyer-common = { path = "../foyer-common" }
+foyer-intrusive = { path = "../foyer-intrusive" }
+foyer-utils = { path = "../foyer-utils" }
+futures = "0.3"
+itertools = "0.10.5"
+lazy_static = "1.4.0"
+libc = "0.2"
+memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = { version = "0.12", features = ["arc_lock"] }
+paste = "1.0"
+prometheus = "0.13"
+rand = "0.8.5"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+tracing = "0.1"
+twox-hash = "1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+hdrhistogram = "7"
+rand_mt = "4.2.1"
+tempfile = "3"
diff --git a/foyer-storage/src/admission.rs b/foyer-storage/src/admission.rs
new file mode 100644
index 00000000..ea500f2c
--- /dev/null
+++ b/foyer-storage/src/admission.rs
@@ -0,0 +1,39 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::marker::PhantomData;
+
+use foyer_common::{Key, Value};
+
+use std::fmt::Debug;
+
+pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
+    type Key: Key;
+    type Value: Value;
+
+    fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
+}
+
+#[derive(Debug, Default)]
+pub struct AdmitAll<K: Key, V: Value>(PhantomData<(K, V)>);
+
+impl<K: Key, V: Value> AdmissionPolicy for AdmitAll<K, V> {
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, _key: &Self::Key, _value: &Self::Value) -> bool {
+        true
+    }
+}
diff --git a/foyer-storage/src/device/error.rs b/foyer-storage/src/device/error.rs
new file mode 100644
index 00000000..17ffc996
--- /dev/null
+++ b/foyer-storage/src/device/error.rs
@@ -0,0 +1,31 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("io error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("nix error: {0}")]
+    Nix(#[from] nix::errno::Errno),
+    #[error("other error: {0}")]
+    Other(String),
+}
+
+impl Error {
+    pub fn io(e: std::io::Error) -> Self {
+        Self::Io(e)
+    }
+}
+
+pub type Result<T> = core::result::Result<T, Error>;
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
new file mode 100644
index 00000000..80ace582
--- /dev/null
+++ b/foyer-storage/src/device/fs.rs
@@ -0,0 +1,266 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fs::{create_dir_all, File, OpenOptions},
+    os::fd::{AsRawFd, RawFd},
+    path::PathBuf,
+    sync::Arc,
+};
+
+use crate::region::RegionId;
+
+use super::{
+    asyncify,
+    error::{Error, Result},
+    io_buffer::AlignedAllocator,
+    Device, IoBuf, IoBufMut,
+};
+use async_trait::async_trait;
+use futures::future::try_join_all;
+use itertools::Itertools;
+
+#[derive(Debug, Clone)]
+pub struct FsDeviceConfig {
+    /// base dir path
+    pub dir: PathBuf,
+
+    /// must be multipliers of `align` and `file_capacity`
+    pub capacity: usize,
+
+    /// must be multipliers of `align`
+    pub file_capacity: usize,
+
+    /// io block alignment, must be pow of 2
+    pub align: usize,
+
+    /// recommended optimized io block size
+    pub io_size: usize,
+}
+
+impl FsDeviceConfig {
+    pub fn verify(&self) {
+        assert!(self.align.is_power_of_two());
+        assert_eq!(self.file_capacity % self.align, 0);
+        assert_eq!(self.capacity % self.file_capacity, 0);
+    }
+}
+
+#[derive(Debug)]
+struct FsDeviceInner {
+    config: FsDeviceConfig,
+
+    dir: File,
+
+    files: Vec<File>,
+
+    io_buffer_allocator: AlignedAllocator,
+}
+
+#[derive(Debug, Clone)]
+pub struct FsDevice {
+    inner: Arc<FsDeviceInner>,
+}
+
+#[async_trait]
+impl Device for FsDevice {
+    type Config = FsDeviceConfig;
+    type IoBufferAllocator = AlignedAllocator;
+
+    async fn open(config: FsDeviceConfig) -> Result<Self> {
+        Self::open(config).await
+    }
+
+    async fn write(
+        &self,
+        buf: impl IoBuf,
+        region: RegionId,
+        offset: u64,
+        len: usize,
+    ) -> Result<()> {
+        assert!(offset as usize + len <= self.inner.config.file_capacity);
+
+        let fd = self.fd(region);
+
+        asyncify(move || {
+            nix::sys::uio::pwrite(fd, &buf.as_ref()[..len], offset as i64)?;
+            Ok(())
+        })
+        .await?;
+
+        Ok(())
+    }
+
+    async fn read(
+        &self,
+        mut buf: impl IoBufMut,
+        region: RegionId,
+        offset: u64,
+        len: usize,
+    ) -> Result<()> {
+        assert!(offset as usize + len <= self.inner.config.file_capacity);
+
+        let fd = self.fd(region);
+
+        asyncify(move || {
+            nix::sys::uio::pread(fd, &mut buf.as_mut()[..len], offset as i64)?;
+            Ok(())
+        })
+        .await?;
+
+        Ok(())
+    }
+
+    async fn flush(&self) -> Result<()> {
+        let fd = self.inner.dir.as_raw_fd();
+        // Commit fs cache to disk. Linux waits for I/O completions.
+        //
+        // See also [syncfs(2)](https://man7.org/linux/man-pages/man2/sync.2.html)
+        #[cfg(target_os = "linux")]
+        asyncify(move || {
+            nix::unistd::syncfs(fd)?;
+            Ok(())
+        })
+        .await?;
+
+        // TODO(MrCroxx): track dirty files and call fsync(2) on them on other target os.
+
+        Ok(())
+    }
+
+    fn capacity(&self) -> usize {
+        self.inner.config.capacity
+    }
+
+    fn regions(&self) -> usize {
+        self.inner.files.len()
+    }
+
+    fn align(&self) -> usize {
+        self.inner.config.align
+    }
+
+    fn io_size(&self) -> usize {
+        self.inner.config.io_size
+    }
+
+    fn io_buffer_allocator(&self) -> &Self::IoBufferAllocator {
+        &self.inner.io_buffer_allocator
+    }
+
+    fn io_buffer(&self, len: usize, capacity: usize) -> Vec<u8, Self::IoBufferAllocator> {
+        assert!(len <= capacity);
+        let mut buf = Vec::with_capacity_in(capacity, self.inner.io_buffer_allocator);
+        unsafe { buf.set_len(len) };
+        buf
+    }
+}
+
+impl FsDevice {
+    pub async fn open(config: FsDeviceConfig) -> Result<Self> {
+        config.verify();
+
+        // TODO(MrCroxx): write and read config to a manifest file for pinning
+
+        let regions = config.capacity / config.file_capacity;
+
+        let path = config.dir.clone();
+        let dir = asyncify(move || {
+            create_dir_all(&path)?;
+            let dir = File::open(&path)?;
+            Ok(dir)
+        })
+        .await?;
+
+        let futures = (0..regions)
+            .map(|i| {
+                let path = config.dir.clone().join(Self::filename(i as RegionId));
+                async move {
+                    let mut opts = OpenOptions::new();
+                    opts.create(true);
+                    opts.write(true);
+                    opts.read(true);
+
+                    // TODO(MrCroxx): use direct i/o on linux targets
+
+                    let file = opts.open(path).map_err(Error::io)?;
+
+                    Ok::<File, Error>(file)
+                }
+            })
+            .collect_vec();
+        let files = try_join_all(futures).await?;
+
+        let io_buffer_allocator = AlignedAllocator::new(config.align);
+
+        let inner = FsDeviceInner {
+            config,
+            dir,
+            files,
+            io_buffer_allocator,
+        };
+
+        Ok(Self {
+            inner: Arc::new(inner),
+        })
+    }
+
+    fn fd(&self, region: RegionId) -> RawFd {
+        self.inner.files[region as usize].as_raw_fd()
+    }
+
+    fn filename(region: RegionId) -> String {
+        format!("foyer-cache-{:08}", region)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use crate::slice::{Slice, SliceMut};
+
+    use super::*;
+
+    const FILES: usize = 8;
+    const FILE_CAPACITY: usize = 8 * 1024; // 8 KiB
+    const CAPACITY: usize = FILES * FILE_CAPACITY; // 64 KiB
+    const ALIGN: usize = 4 * 1024;
+
+    #[tokio::test]
+    async fn test_fs_device_simple() {
+        let dir = tempfile::tempdir().unwrap();
+        let config = FsDeviceConfig {
+            dir: PathBuf::from(dir.path()),
+            capacity: CAPACITY,
+            file_capacity: FILE_CAPACITY,
+            align: ALIGN,
+            io_size: ALIGN,
+        };
+        let dev = FsDevice::open(config).await.unwrap();
+
+        let wbuffer = vec![b'x'; ALIGN];
+        let mut rbuffer = vec![0; ALIGN];
+
+        let wbuf = unsafe { Slice::new(&wbuffer) };
+        let rbuf = unsafe { SliceMut::new(&mut rbuffer) };
+
+        dev.write(wbuf, 0, 0, ALIGN).await.unwrap();
+        dev.read(rbuf, 0, 0, ALIGN).await.unwrap();
+
+        assert_eq!(&wbuffer, &rbuffer);
+
+        drop(wbuffer);
+        drop(rbuffer);
+    }
+}
diff --git a/foyer-storage/src/device/io_buffer.rs b/foyer-storage/src/device/io_buffer.rs
new file mode 100644
index 00000000..9046caec
--- /dev/null
+++ b/foyer-storage/src/device/io_buffer.rs
@@ -0,0 +1,73 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use foyer_utils::bits;
+use std::alloc::{Allocator, Global};
+
+#[derive(Debug, Clone, Copy)]
+pub struct AlignedAllocator {
+    align: usize,
+}
+
+impl AlignedAllocator {
+    pub const fn new(align: usize) -> Self {
+        assert!(align.is_power_of_two());
+        Self { align }
+    }
+}
+
+unsafe impl Allocator for AlignedAllocator {
+    fn allocate(
+        &self,
+        layout: std::alloc::Layout,
+    ) -> Result<std::ptr::NonNull<[u8]>, std::alloc::AllocError> {
+        let layout = std::alloc::Layout::from_size_align(
+            layout.size(),
+            bits::align_up(self.align, layout.align()),
+        )
+        .unwrap();
+        Global.allocate(layout)
+    }
+
+    unsafe fn deallocate(&self, ptr: std::ptr::NonNull<u8>, layout: std::alloc::Layout) {
+        let layout = std::alloc::Layout::from_size_align(
+            layout.size(),
+            bits::align_up(self.align, layout.align()),
+        )
+        .unwrap();
+        Global.deallocate(ptr, layout)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_aligned_buffer() {
+        const ALIGN: usize = 512;
+        let allocator = AlignedAllocator::new(ALIGN);
+
+        let mut buf: Vec<u8, _> = Vec::with_capacity_in(ALIGN * 8, &allocator);
+        bits::assert_aligned(ALIGN, buf.as_ptr().addr());
+
+        buf.extend_from_slice(&[b'x'; ALIGN * 8]);
+        bits::assert_aligned(ALIGN, buf.as_ptr().addr());
+        assert_eq!(buf, [b'x'; ALIGN * 8]);
+
+        buf.extend_from_slice(&[b'x'; ALIGN * 8]);
+        bits::assert_aligned(ALIGN, buf.as_ptr().addr());
+        assert_eq!(buf, [b'x'; ALIGN * 16])
+    }
+}
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
new file mode 100644
index 00000000..a8d65270
--- /dev/null
+++ b/foyer-storage/src/device/mod.rs
@@ -0,0 +1,155 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+pub mod error;
+pub mod fs;
+pub mod io_buffer;
+
+use async_trait::async_trait;
+use error::Result;
+
+use std::{alloc::Allocator, fmt::Debug};
+
+use crate::region::RegionId;
+
+pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
+pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
+pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
+
+#[async_trait]
+pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
+    type IoBufferAllocator: BufferAllocator;
+    type Config: Debug;
+
+    async fn open(config: Self::Config) -> Result<Self>;
+
+    async fn write(&self, buf: impl IoBuf, region: RegionId, offset: u64, len: usize)
+        -> Result<()>;
+
+    async fn read(
+        &self,
+        buf: impl IoBufMut,
+        region: RegionId,
+        offset: u64,
+        len: usize,
+    ) -> Result<()>;
+
+    async fn flush(&self) -> Result<()>;
+
+    fn capacity(&self) -> usize;
+
+    fn regions(&self) -> usize;
+
+    fn align(&self) -> usize;
+
+    fn io_size(&self) -> usize;
+
+    fn io_buffer_allocator(&self) -> &Self::IoBufferAllocator;
+
+    fn io_buffer(&self, len: usize, capacity: usize) -> Vec<u8, Self::IoBufferAllocator>;
+
+    fn region_size(&self) -> usize {
+        debug_assert!(self.capacity() % self.regions() == 0);
+        self.capacity() / self.regions()
+    }
+}
+
+async fn asyncify<F, T>(f: F) -> error::Result<T>
+where
+    F: FnOnce() -> error::Result<T> + Send + 'static,
+    T: Send + 'static,
+{
+    #[cfg_attr(madsim, expect(deprecated))]
+    match tokio::task::spawn_blocking(f).await {
+        Ok(res) => res,
+        Err(e) => Err(error::Error::Other(format!(
+            "background task failed: {:?}",
+            e,
+        ))),
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::{io_buffer::AlignedAllocator, *};
+
+    #[derive(Debug, Clone)]
+    pub struct NullDevice(AlignedAllocator);
+
+    impl NullDevice {
+        pub fn new(align: usize) -> Self {
+            Self(AlignedAllocator::new(align))
+        }
+    }
+
+    #[async_trait]
+    impl Device for NullDevice {
+        type Config = usize;
+        type IoBufferAllocator = AlignedAllocator;
+
+        async fn open(config: usize) -> Result<Self> {
+            Ok(Self::new(config))
+        }
+
+        async fn write(
+            &self,
+            _buf: impl IoBuf,
+            _region: RegionId,
+            _offset: u64,
+            _len: usize,
+        ) -> Result<()> {
+            Ok(())
+        }
+
+        async fn read(
+            &self,
+            _buf: impl IoBufMut,
+            _region: RegionId,
+            _offset: u64,
+            _len: usize,
+        ) -> Result<()> {
+            Ok(())
+        }
+
+        async fn flush(&self) -> Result<()> {
+            Ok(())
+        }
+
+        fn capacity(&self) -> usize {
+            usize::MAX
+        }
+
+        fn regions(&self) -> usize {
+            4096
+        }
+
+        fn align(&self) -> usize {
+            4096
+        }
+
+        fn io_size(&self) -> usize {
+            4096
+        }
+
+        fn io_buffer_allocator(&self) -> &Self::IoBufferAllocator {
+            &self.0
+        }
+
+        fn io_buffer(&self, len: usize, capacity: usize) -> Vec<u8, Self::IoBufferAllocator> {
+            let mut buf = Vec::with_capacity_in(capacity, self.0);
+            unsafe { buf.set_len(len) };
+            buf
+        }
+    }
+}
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
new file mode 100644
index 00000000..7ebc812b
--- /dev/null
+++ b/foyer-storage/src/error.rs
@@ -0,0 +1,33 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("device error: {0}")]
+    Device(#[from] super::device::error::Error),
+    #[error("entry too large: {len} > {capacity}")]
+    EntryTooLarge { len: usize, capacity: usize },
+    #[error("checksum mismatch, checksum: {checksum}, expected: {expected}")]
+    ChecksumMismatch { checksum: u64, expected: u64 },
+    #[error("other error: {0}")]
+    Other(String),
+}
+
+impl Error {
+    pub fn device(e: super::device::error::Error) -> Self {
+        Self::Device(e)
+    }
+}
+
+pub type Result<T> = core::result::Result<T, Error>;
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
new file mode 100644
index 00000000..09d1a8ed
--- /dev/null
+++ b/foyer-storage/src/flusher.rs
@@ -0,0 +1,181 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use foyer_utils::queue::AsyncQueue;
+use itertools::Itertools;
+use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+
+use crate::{
+    device::{BufferAllocator, Device},
+    error::Result,
+    region::RegionId,
+    region_manager::{RegionEpItemAdapter, RegionManager},
+    slice::Slice,
+};
+
+#[derive(Debug)]
+pub struct FlushTask {
+    pub region_id: RegionId,
+}
+
+pub struct Flusher {
+    sequence: AtomicUsize,
+
+    task_txs: Vec<UnboundedSender<FlushTask>>,
+}
+
+impl Flusher {
+    pub fn new(runners: usize) -> Self {
+        Self {
+            sequence: AtomicUsize::new(0),
+            task_txs: Vec::with_capacity(runners),
+        }
+    }
+
+    pub fn run<A, D, E, EL>(
+        &mut self,
+        buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+        region_manager: Arc<RegionManager<A, D, E, EL>>,
+    ) where
+        A: BufferAllocator,
+        D: Device<IoBufferAllocator = A>,
+        E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+        EL: Link,
+    {
+        let runners = self.task_txs.capacity();
+
+        #[allow(clippy::type_complexity)]
+        let (mut txs, rxs): (
+            Vec<UnboundedSender<FlushTask>>,
+            Vec<UnboundedReceiver<FlushTask>>,
+        ) = (0..runners).map(|_| unbounded_channel()).unzip();
+        self.task_txs.append(&mut txs);
+
+        let runners = rxs
+            .into_iter()
+            .map(|rx| Runner {
+                task_rx: rx,
+                buffers: buffers.clone(),
+                region_manager: region_manager.clone(),
+            })
+            .collect_vec();
+
+        for runner in runners {
+            tokio::spawn(async move {
+                runner.run().await.unwrap();
+            });
+        }
+    }
+
+    pub fn submit(&self, task: FlushTask) {
+        let submittee = self.sequence.fetch_add(1, Ordering::Relaxed) % self.task_txs.len();
+        self.task_txs[submittee].send(task).unwrap();
+    }
+}
+
+struct Runner<A, D, E, EL>
+where
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    task_rx: UnboundedReceiver<FlushTask>,
+    buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+
+    region_manager: Arc<RegionManager<A, D, E, EL>>,
+}
+
+impl<A, D, E, EL> Runner<A, D, E, EL>
+where
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    async fn run(mut self) -> Result<()> {
+        loop {
+            if let Some(task) = self.task_rx.recv().await {
+                // TODO(MrCroxx): seal buffer
+
+                tracing::info!("[flusher] receive flush task, region: {}", task.region_id);
+
+                let region = self.region_manager.region(&task.region_id);
+
+                {
+                    // step 1: write buffer back to device
+                    let slice = region.load(.., 0).await?.unwrap();
+
+                    // wait all physical readers (from previous version) and writers done
+                    let guard = region.exclusive(false, true, false).await;
+
+                    tracing::info!("[flusher] write region {} back to device", task.region_id);
+
+                    let mut offset = 0;
+                    let len = region.device().io_size();
+                    while offset < region.device().region_size() {
+                        let start = offset;
+                        let end = std::cmp::min(offset + len, region.device().region_size());
+
+                        tracing::trace!("write region {} [{}..{}]", region.id(), start, end);
+
+                        let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
+                        region
+                            .device()
+                            .write(s, region.id(), offset as u64, len)
+                            .await?;
+                        offset += len;
+                    }
+
+                    drop(guard);
+
+                    tracing::debug!("[flusher] drop exclusive guard");
+                }
+
+                let buffer = {
+                    // step 2: detach buffer
+                    let mut guard = region.exclusive(false, false, true).await;
+
+                    let buffer = guard.detach_buffer();
+
+                    tracing::trace!(
+                            "[flusher] region {}, writers: {}, buffered readers: {}, physical readers: {}",
+                            region.id(),
+                            guard.writers(),
+                            guard.buffered_readers(),
+                            guard.physical_readers()
+                        );
+
+                    drop(guard);
+                    buffer
+                };
+
+                // step 3: release buffer
+                self.buffers.release(buffer);
+
+                self.region_manager.post_flush(&region.id()).await;
+
+                tracing::info!("[flusher] finish flush task, region: {}", task.region_id);
+            } else {
+                return Ok(());
+            }
+        }
+    }
+}
diff --git a/foyer-storage/src/indices.rs b/foyer-storage/src/indices.rs
new file mode 100644
index 00000000..9c87641c
--- /dev/null
+++ b/foyer-storage/src/indices.rs
@@ -0,0 +1,110 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::collections::BTreeMap;
+
+use foyer_common::Key;
+use itertools::Itertools;
+use parking_lot::RwLock;
+
+use crate::region::{RegionId, Version};
+
+#[derive(Debug, Clone)]
+pub struct Index<K>
+where
+    K: Key,
+{
+    pub key: K,
+
+    pub region: RegionId,
+    pub version: Version,
+    pub offset: u32,
+    pub len: u32,
+    pub key_len: u32,
+    pub value_len: u32,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct Slot {
+    pub region: RegionId,
+    pub sequence: u32,
+}
+
+#[derive(Debug)]
+struct IndicesInner<K>
+where
+    K: Key,
+{
+    slots: BTreeMap<K, Slot>,
+    regions: Vec<BTreeMap<u32, Index<K>>>,
+}
+
+#[derive(Debug)]
+pub struct Indices<K>
+where
+    K: Key,
+{
+    inner: RwLock<IndicesInner<K>>,
+}
+
+impl<K> Indices<K>
+where
+    K: Key,
+{
+    pub fn new(regions: usize) -> Self {
+        let inner = IndicesInner {
+            slots: BTreeMap::new(),
+            regions: vec![BTreeMap::new(); regions],
+        };
+        Self {
+            inner: RwLock::new(inner),
+        }
+    }
+
+    pub fn insert(&self, index: Index<K>) {
+        let region = index.region;
+        let key = index.key.clone();
+
+        let mut inner = self.inner.write();
+        let sequence = inner.regions[region as usize].len() as u32;
+        inner.regions[region as usize].insert(sequence, index);
+        inner.slots.insert(key, Slot { region, sequence });
+    }
+
+    pub fn lookup(&self, key: &K) -> Option<Index<K>> {
+        let inner = self.inner.read();
+        let slot = inner.slots.get(key)?;
+        inner.regions[slot.region as usize]
+            .get(&slot.sequence)
+            .cloned()
+    }
+
+    pub fn remove(&self, key: &K) -> Option<Index<K>> {
+        let mut inner = self.inner.write();
+        let slot = inner.slots.remove(key)?;
+        inner.regions[slot.region as usize].remove(&slot.sequence)
+    }
+
+    pub fn take_region(&self, region: &RegionId) -> Vec<Index<K>> {
+        let mut inner = self.inner.write();
+        let mut indices = BTreeMap::new();
+        std::mem::swap(&mut indices, &mut inner.regions[*region as usize]);
+
+        for index in indices.values() {
+            inner.slots.remove(&index.key);
+        }
+
+        indices.into_values().collect_vec()
+    }
+}
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
new file mode 100644
index 00000000..d969ee22
--- /dev/null
+++ b/foyer-storage/src/lib.rs
@@ -0,0 +1,31 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(allocator_api)]
+#![feature(strict_provenance)]
+#![feature(trait_alias)]
+#![feature(get_mut_unchecked)]
+#![allow(clippy::type_complexity)]
+
+pub mod admission;
+pub mod device;
+pub mod error;
+pub mod flusher;
+pub mod indices;
+pub mod reclaimer;
+pub mod region;
+pub mod region_manager;
+pub mod reinsertion;
+pub mod slice;
+pub mod store;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
new file mode 100644
index 00000000..af6c9612
--- /dev/null
+++ b/foyer-storage/src/reclaimer.rs
@@ -0,0 +1,178 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+use crate::{
+    admission::AdmissionPolicy,
+    device::{BufferAllocator, Device},
+    error::Result,
+    indices::Indices,
+    region::RegionId,
+    region_manager::{RegionEpItemAdapter, RegionManager},
+    reinsertion::ReinsertionPolicy,
+    store::Store,
+};
+use foyer_common::{Key, Value};
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use foyer_utils::queue::AsyncQueue;
+use itertools::Itertools;
+use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+
+#[derive(Debug)]
+pub struct ReclaimTask {
+    pub region_id: RegionId,
+}
+
+pub struct Reclaimer {
+    sequence: AtomicUsize,
+
+    task_txs: Vec<UnboundedSender<ReclaimTask>>,
+}
+
+impl Reclaimer {
+    pub fn new(runners: usize) -> Self {
+        Self {
+            sequence: AtomicUsize::new(0),
+            task_txs: Vec::with_capacity(runners),
+        }
+    }
+
+    pub fn run<K, V, A, D, EP, AP, RP, EL>(
+        &mut self,
+        store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
+        region_manager: Arc<RegionManager<A, D, EP, EL>>,
+        clean_regions: Arc<AsyncQueue<RegionId>>,
+        reinsertion: RP,
+        indices: Arc<Indices<K>>,
+    ) where
+        K: Key,
+        V: Value,
+        A: BufferAllocator,
+        D: Device<IoBufferAllocator = A>,
+        EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+        AP: AdmissionPolicy<Key = K, Value = V>,
+        RP: ReinsertionPolicy<Key = K, Value = V>,
+        EL: Link,
+    {
+        let runners = self.task_txs.capacity();
+
+        #[allow(clippy::type_complexity)]
+        let (mut txs, rxs): (
+            Vec<UnboundedSender<ReclaimTask>>,
+            Vec<UnboundedReceiver<ReclaimTask>>,
+        ) = (0..runners).map(|_| unbounded_channel()).unzip();
+        self.task_txs.append(&mut txs);
+
+        let runners = rxs
+            .into_iter()
+            .map(|rx| Runner {
+                task_rx: rx,
+                _store: store.clone(),
+                region_manager: region_manager.clone(),
+                clean_regions: clean_regions.clone(),
+                _reinsertion: reinsertion.clone(),
+                indices: indices.clone(),
+            })
+            .collect_vec();
+
+        for runner in runners {
+            tokio::spawn(async move {
+                runner.run().await.unwrap();
+            });
+        }
+    }
+
+    pub fn submit(&self, task: ReclaimTask) {
+        let submittee = self.sequence.fetch_add(1, Ordering::Relaxed) % self.task_txs.len();
+        self.task_txs[submittee].send(task).unwrap();
+    }
+}
+
+struct Runner<K, V, A, D, EP, AP, RP, EL>
+where
+    K: Key,
+    V: Value,
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    AP: AdmissionPolicy<Key = K, Value = V>,
+    RP: ReinsertionPolicy<Key = K, Value = V>,
+    EL: Link,
+{
+    task_rx: UnboundedReceiver<ReclaimTask>,
+
+    _store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
+    region_manager: Arc<RegionManager<A, D, EP, EL>>,
+    clean_regions: Arc<AsyncQueue<RegionId>>,
+    _reinsertion: RP,
+    indices: Arc<Indices<K>>,
+}
+
+impl<K, V, A, D, EP, AP, RP, EL> Runner<K, V, A, D, EP, AP, RP, EL>
+where
+    K: Key,
+    V: Value,
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    AP: AdmissionPolicy<Key = K, Value = V>,
+    RP: ReinsertionPolicy<Key = K, Value = V>,
+    EL: Link,
+{
+    async fn run(mut self) -> Result<()> {
+        loop {
+            if let Some(task) = self.task_rx.recv().await {
+                tracing::info!(
+                    "[reclaimer] receive reclaim task, region: {}",
+                    task.region_id
+                );
+
+                let region = self.region_manager.region(&task.region_id);
+
+                // keep region totally exclusive while reclamation
+                let guard = region.exclusive(false, false, false).await;
+
+                tracing::trace!(
+                    "[reclaimer] region {}, writers: {}, buffered readers: {}, physical readers: {}",
+                    region.id(),
+                    guard.writers(),
+                    guard.buffered_readers(),
+                    guard.physical_readers()
+                );
+
+                // step 1: drop indices
+                let _indices = self.indices.take_region(&task.region_id);
+
+                // step 2: do reinsertion
+                // TODO(MrCroxx): do reinsertion
+
+                // step 3: send clean region
+                self.clean_regions.release(task.region_id);
+
+                drop(guard);
+
+                tracing::info!(
+                    "[reclaimer] finish reclaim task, region: {}",
+                    task.region_id
+                );
+            } else {
+                return Ok(());
+            }
+        }
+    }
+}
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
new file mode 100644
index 00000000..6cc4ed32
--- /dev/null
+++ b/foyer-storage/src/region.rs
@@ -0,0 +1,509 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fmt::Debug,
+    ops::{Deref, DerefMut, RangeBounds},
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll, Waker},
+};
+
+use futures::Future;
+use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock};
+
+use crate::{
+    device::{BufferAllocator, Device},
+    error::Result,
+    slice::{Slice, SliceMut},
+};
+
+pub type RegionId = u32;
+/// 0 matches any version
+pub type Version = u32;
+
+#[derive(Debug)]
+pub struct RegionInner<A>
+where
+    A: BufferAllocator,
+{
+    version: Version,
+
+    buffer: Option<Vec<u8, A>>,
+    len: usize,
+    capacity: usize,
+
+    writers: usize,
+    buffered_readers: usize,
+    physical_readers: usize,
+
+    wakers: Vec<Waker>,
+}
+
+#[derive(Debug)]
+pub struct Region<A, D>
+where
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+{
+    id: RegionId,
+
+    inner: Arc<RwLock<RegionInner<A>>>,
+
+    device: D,
+}
+
+/// [`Region`] represents a contiguous aligned range on device and its optional dirty buffer.
+///
+/// [`Region`] may be in one of the following states:
+///
+/// - buffered write : append-only buffer write, written parts can be read concurrently.
+/// - buffered read  : happenes if the region is dirty with an attached dirty buffer
+/// - physical read  : happenes if the region is clean, read directly from the devie
+/// - flush          : happenes after the region dirty buffer is full, there are 2 steps when flushing
+///                    step 1 writes dirty buffer to device, must guarantee there is no writers or physical readers
+///                    step 2 detaches dirty buffer, must guarantee there is no buffer readers
+/// - reclaim        : happens after the region is evicted, must guarantee there is no writers, buffer readers or physical readers,
+///                    *or in-flight writers or readers* (verify by version)
+impl<A, D> Region<A, D>
+where
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+{
+    pub fn new(id: RegionId, device: D) -> Self {
+        let inner = RegionInner {
+            version: 1,
+
+            buffer: None,
+            len: 0,
+            capacity: device.region_size(),
+
+            writers: 0,
+            buffered_readers: 0,
+            physical_readers: 0,
+
+            wakers: vec![],
+        };
+        Self {
+            id,
+            inner: Arc::new(RwLock::new(inner)),
+            device,
+        }
+    }
+
+    pub fn allocate(&self, size: usize) -> Option<WriteSlice> {
+        let callback = {
+            let inner = self.inner.clone();
+            move || {
+                let mut guard = inner.write();
+                guard.writers -= 1;
+                guard.wake_all();
+            }
+        };
+
+        let mut inner = self.inner.write();
+
+        if inner.len + size > inner.capacity {
+            return None;
+        }
+
+        inner.writers += 1;
+        let version = inner.version;
+        let offset = inner.len;
+        inner.len += size;
+        let buffer = inner.buffer.as_mut().unwrap();
+
+        let slice = unsafe { SliceMut::new(&mut buffer[offset..offset + size]) };
+        let region_id = self.id;
+
+        let slice = WriteSlice {
+            slice,
+            region_id,
+            version,
+            offset,
+            callback: Box::new(callback),
+        };
+
+        Some(slice)
+    }
+
+    pub async fn load(
+        &self,
+        range: impl RangeBounds<usize>,
+        version: Version,
+    ) -> Result<Option<ReadSlice<A>>> {
+        let start = match range.start_bound() {
+            std::ops::Bound::Included(i) => *i,
+            std::ops::Bound::Excluded(i) => *i + 1,
+            std::ops::Bound::Unbounded => 0,
+        };
+        let end = match range.end_bound() {
+            std::ops::Bound::Included(i) => *i + 1,
+            std::ops::Bound::Excluded(i) => *i,
+            std::ops::Bound::Unbounded => self.device.region_size(),
+        };
+
+        // restrict guard lifetime
+        {
+            let mut inner = self.inner.write();
+            if version != 0 && version != inner.version {
+                return Ok(None);
+            }
+
+            // if buffer attached, buffered read
+
+            if inner.buffer.is_some() {
+                inner.buffered_readers += 1;
+                let allocator = inner.buffer.as_ref().unwrap().allocator().clone();
+                let slice = unsafe { Slice::new(&inner.buffer.as_ref().unwrap()[start..end]) };
+                let callback = {
+                    let inner = self.inner.clone();
+                    move || {
+                        let mut guard = inner.write();
+                        guard.buffered_readers -= 1;
+                        guard.wake_all();
+                    }
+                };
+                return Ok(Some(ReadSlice::Slice {
+                    slice,
+                    allocator: Some(allocator),
+                    callback: Box::new(callback),
+                }));
+            }
+
+            // if buffer detached, physical read
+            inner.physical_readers += 1;
+            drop(inner);
+        }
+
+        let region = self.id;
+        let offset = start;
+        let len = end - start;
+
+        tracing::trace!(
+            "read from disk, region: {}, offset: {}, len: {}",
+            region,
+            offset,
+            len
+        );
+
+        let mut buf = self.device.io_buffer(len, len);
+        let slice = unsafe { SliceMut::new(&mut buf) };
+        self.device.read(slice, region, offset as u64, len).await?;
+        let callback = {
+            let inner = self.inner.clone();
+            move || {
+                let mut guard = inner.write();
+                guard.physical_readers -= 1;
+                guard.wake_all();
+            }
+        };
+        Ok(Some(ReadSlice::Owned {
+            buf: Some(buf),
+            callback: Box::new(callback),
+        }))
+    }
+
+    pub fn attach_buffer(&self, buf: Vec<u8, A>) {
+        let mut inner = self.inner.write();
+
+        assert_eq!(inner.writers, 0);
+        assert_eq!(inner.buffered_readers, 0);
+
+        inner.attach_buffer(buf);
+    }
+
+    pub fn detach_buffer(&self) -> Vec<u8, A> {
+        let mut inner = self.inner.write();
+
+        inner.detach_buffer()
+    }
+
+    pub fn has_buffer(&self) -> bool {
+        let inner = self.inner.read();
+        inner.has_buffer()
+    }
+
+    pub async fn exclusive(
+        &self,
+        can_write: bool,
+        can_buffered_read: bool,
+        can_physical_read: bool,
+    ) -> ExclusiveGuard<A> {
+        ExclusiveFuture {
+            inner: self.inner.clone(),
+            can_write,
+            can_buffered_read,
+            can_physical_read,
+            is_waker_set: false,
+        }
+        .await
+    }
+
+    pub fn id(&self) -> RegionId {
+        self.id
+    }
+
+    pub fn device(&self) -> &D {
+        &self.device
+    }
+
+    pub fn version(&self) -> Version {
+        self.inner.read().version
+    }
+
+    pub fn advance(&self) -> Version {
+        let mut inner = self.inner.write();
+        let res = inner.version;
+        inner.version += 1;
+        res
+    }
+}
+
+impl<A> RegionInner<A>
+where
+    A: BufferAllocator,
+{
+    pub fn attach_buffer(&mut self, buf: Vec<u8, A>) {
+        assert!(self.buffer.is_none());
+        assert_eq!(buf.len(), buf.capacity());
+        assert_eq!(buf.capacity(), self.capacity);
+        self.buffer = Some(buf);
+        self.len = 0;
+    }
+
+    pub fn detach_buffer(&mut self) -> Vec<u8, A> {
+        self.buffer.take().unwrap()
+    }
+
+    pub fn has_buffer(&self) -> bool {
+        self.buffer.is_some()
+    }
+
+    pub fn writers(&self) -> usize {
+        self.writers
+    }
+
+    pub fn buffered_readers(&self) -> usize {
+        self.buffered_readers
+    }
+
+    pub fn physical_readers(&self) -> usize {
+        self.physical_readers
+    }
+
+    fn wake_all(&self) {
+        for waker in self.wakers.iter() {
+            waker.wake_by_ref();
+        }
+    }
+}
+
+// future
+
+pub struct ExclusiveGuard<A: BufferAllocator> {
+    inner: ArcRwLockWriteGuard<RawRwLock, RegionInner<A>>,
+}
+
+unsafe impl<A: BufferAllocator> Send for ExclusiveGuard<A> {}
+
+impl<A: BufferAllocator> Deref for ExclusiveGuard<A> {
+    type Target = RegionInner<A>;
+
+    fn deref(&self) -> &Self::Target {
+        self.inner.deref()
+    }
+}
+
+impl<A: BufferAllocator> DerefMut for ExclusiveGuard<A> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.inner.deref_mut()
+    }
+}
+
+struct ExclusiveFuture<A>
+where
+    A: BufferAllocator,
+{
+    inner: Arc<RwLock<RegionInner<A>>>,
+
+    can_write: bool,
+    can_buffered_read: bool,
+    can_physical_read: bool,
+
+    is_waker_set: bool,
+}
+
+impl<A: BufferAllocator> ExclusiveFuture<A> {
+    fn is_ready(&self, guard: &ArcRwLockWriteGuard<RawRwLock, RegionInner<A>>) -> bool {
+        tracing::trace!("exclusive: [can write: {}, writers: {}] [can buffered read: {}, buffered readers: {}] [can physical read: {}, physical readers: {}]", self.can_write, guard.writers, self.can_buffered_read, guard.buffered_readers, self.can_physical_read, guard.physical_readers);
+        (self.can_write || guard.writers == 0)
+            && (self.can_buffered_read || guard.buffered_readers == 0)
+            && (self.can_physical_read || guard.physical_readers == 0)
+    }
+}
+
+impl<A: BufferAllocator> Future for ExclusiveFuture<A> {
+    type Output = ExclusiveGuard<A>;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let inner = self.inner.clone();
+        let mut guard = inner.write_arc();
+        let is_ready = self.is_ready(&guard);
+        if is_ready {
+            Poll::Ready(ExclusiveGuard { inner: guard })
+        } else {
+            if !self.is_waker_set {
+                self.is_waker_set = true;
+                guard.wakers.push(cx.waker().clone());
+            }
+            Poll::Pending
+        }
+    }
+}
+// read & write slice
+
+pub type DropCallback = Box<dyn Fn() + Send + Sync + 'static>;
+
+pub struct WriteSlice {
+    slice: SliceMut,
+    region_id: RegionId,
+    version: Version,
+    offset: usize,
+    callback: DropCallback,
+}
+
+impl Debug for WriteSlice {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("WriteSlice")
+            .field("slice", &self.slice)
+            .field("region_id", &self.region_id)
+            .field("version", &self.version)
+            .field("offset", &self.offset)
+            .finish()
+    }
+}
+
+impl WriteSlice {
+    pub fn region_id(&self) -> RegionId {
+        self.region_id
+    }
+
+    pub fn version(&self) -> Version {
+        self.version
+    }
+
+    pub fn offset(&self) -> usize {
+        self.offset
+    }
+
+    pub fn len(&self) -> usize {
+        self.slice.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+impl AsRef<[u8]> for WriteSlice {
+    fn as_ref(&self) -> &[u8] {
+        self.slice.as_ref()
+    }
+}
+
+impl AsMut<[u8]> for WriteSlice {
+    fn as_mut(&mut self) -> &mut [u8] {
+        self.slice.as_mut()
+    }
+}
+
+impl Drop for WriteSlice {
+    fn drop(&mut self) {
+        (self.callback)();
+    }
+}
+
+pub enum ReadSlice<A>
+where
+    A: BufferAllocator,
+{
+    Slice {
+        slice: Slice,
+        allocator: Option<A>,
+        callback: DropCallback,
+    },
+    Owned {
+        buf: Option<Vec<u8, A>>,
+        callback: DropCallback,
+    },
+}
+
+impl<A> Debug for ReadSlice<A>
+where
+    A: BufferAllocator,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Slice {
+                slice, allocator, ..
+            } => f
+                .debug_struct("Slice")
+                .field("slice", slice)
+                .field("allocator", allocator)
+                .finish(),
+            Self::Owned { buf, .. } => f.debug_struct("Owned").field("buf", buf).finish(),
+        }
+    }
+}
+
+impl<A> ReadSlice<A>
+where
+    A: BufferAllocator,
+{
+    pub fn len(&self) -> usize {
+        match self {
+            Self::Slice { slice, .. } => slice.len(),
+            Self::Owned { buf, .. } => buf.as_ref().unwrap().len(),
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+impl<A> AsRef<[u8]> for ReadSlice<A>
+where
+    A: BufferAllocator,
+{
+    fn as_ref(&self) -> &[u8] {
+        match self {
+            Self::Slice { slice, .. } => slice.as_ref(),
+            Self::Owned { buf, .. } => buf.as_ref().unwrap(),
+        }
+    }
+}
+
+impl<A> Drop for ReadSlice<A>
+where
+    A: BufferAllocator,
+{
+    fn drop(&mut self) {
+        match self {
+            Self::Slice { callback, .. } => (callback)(),
+            Self::Owned { callback, .. } => (callback)(),
+        }
+    }
+}
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
new file mode 100644
index 00000000..aed6626c
--- /dev/null
+++ b/foyer-storage/src/region_manager.rs
@@ -0,0 +1,181 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{marker::PhantomData, sync::Arc};
+
+use foyer_intrusive::{
+    core::{adapter::Link, pointer::PointerOps},
+    eviction::EvictionPolicy,
+    intrusive_adapter, key_adapter,
+};
+use foyer_utils::queue::AsyncQueue;
+use tokio::sync::RwLock;
+
+use crate::{
+    device::{BufferAllocator, Device},
+    flusher::{FlushTask, Flusher},
+    reclaimer::{ReclaimTask, Reclaimer},
+    region::{Region, RegionId, WriteSlice},
+};
+
+#[derive(Debug)]
+pub struct RegionEpItem<L>
+where
+    L: Link,
+{
+    link: L,
+    id: RegionId,
+}
+
+intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEpItem<L> { link: L } where L: Link }
+key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
+
+struct RegionManagerInner<A, E, EL>
+where
+    A: BufferAllocator,
+    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    current: Option<RegionId>,
+
+    buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+    clean_regions: Arc<AsyncQueue<RegionId>>,
+
+    eviction: E,
+
+    _marker: PhantomData<EL>,
+}
+
+pub struct RegionManager<A, D, E, EL>
+where
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    inner: Arc<RwLock<RegionManagerInner<A, E, EL>>>,
+
+    regions: Vec<Region<A, D>>,
+    items: Vec<Arc<RegionEpItem<EL>>>,
+
+    flusher: Arc<Flusher>,
+    reclaimer: Arc<Reclaimer>,
+}
+
+impl<A, D, E, EL> RegionManager<A, D, E, EL>
+where
+    A: BufferAllocator,
+    D: Device<IoBufferAllocator = A>,
+    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    pub fn new(
+        region_nums: usize,
+        eviction_config: E::Config,
+        buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+        clean_regions: Arc<AsyncQueue<RegionId>>,
+        device: D,
+        flusher: Arc<Flusher>,
+        reclaimer: Arc<Reclaimer>,
+    ) -> Self {
+        let eviction = E::new(eviction_config);
+
+        let mut regions = Vec::with_capacity(region_nums);
+        let mut items = Vec::with_capacity(region_nums);
+
+        for id in 0..region_nums as RegionId {
+            let region = Region::new(id, device.clone());
+            let item = Arc::new(RegionEpItem {
+                link: E::Link::default(),
+                id,
+            });
+
+            regions.push(region);
+            items.push(item);
+        }
+
+        let inner = RegionManagerInner {
+            current: None,
+            buffers,
+            clean_regions,
+            eviction,
+            _marker: PhantomData,
+        };
+
+        Self {
+            inner: Arc::new(RwLock::new(inner)),
+            regions,
+            items,
+            flusher,
+            reclaimer,
+        }
+    }
+
+    /// Allocate a buffer slice with given size in an active region to write.
+    pub async fn allocate(&self, size: usize) -> WriteSlice {
+        let mut inner = self.inner.write().await;
+
+        // try allocate from current region
+        if let Some(region_id) = inner.current {
+            let region = self.region(&region_id);
+            if let Some(slice) = region.allocate(size) {
+                return slice;
+            }
+
+            // current region is full, schedule flushing
+            self.flusher.submit(FlushTask { region_id });
+            inner.current = None;
+        }
+
+        assert!(inner.current.is_none());
+
+        tracing::debug!("clean regions: {}", inner.clean_regions.len());
+        if inner.clean_regions.is_empty() {
+            if let Some(item) = inner.eviction.pop() {
+                self.reclaimer.submit(ReclaimTask { region_id: item.id });
+            }
+        }
+
+        let region_id = inner.clean_regions.acquire().await;
+        tracing::info!("switch to clean region: {}", region_id);
+
+        let region = self.region(&region_id);
+        let buffer = inner.buffers.acquire().await;
+        region.attach_buffer(buffer);
+
+        let slice = region.allocate(size).unwrap();
+
+        region.advance();
+        inner.current = Some(region_id);
+
+        slice
+    }
+
+    pub fn region(&self, id: &RegionId) -> &Region<A, D> {
+        &self.regions[*id as usize]
+    }
+
+    pub async fn record_access(&self, id: &RegionId) {
+        let mut inner = self.inner.write().await;
+        let item = &self.items[*id as usize];
+        if item.link.is_linked() {
+            inner.eviction.access(&self.items[*id as usize]);
+        }
+    }
+
+    pub async fn post_flush(&self, id: &RegionId) {
+        let mut inner = self.inner.write().await;
+        inner.eviction.push(self.items[*id as usize].clone());
+    }
+}
diff --git a/foyer-storage/src/reinsertion.rs b/foyer-storage/src/reinsertion.rs
new file mode 100644
index 00000000..49b932b8
--- /dev/null
+++ b/foyer-storage/src/reinsertion.rs
@@ -0,0 +1,45 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::marker::PhantomData;
+
+use foyer_common::{Key, Value};
+
+use std::fmt::Debug;
+
+pub trait ReinsertionPolicy: Send + Sync + 'static + Clone + Debug {
+    type Key: Key;
+    type Value: Value;
+
+    fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
+}
+
+#[derive(Debug, Default)]
+pub struct ReinsertNone<K: Key, V: Value>(PhantomData<(K, V)>);
+
+impl<K: Key, V: Value> Clone for ReinsertNone<K, V> {
+    fn clone(&self) -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<K: Key, V: Value> ReinsertionPolicy for ReinsertNone<K, V> {
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, _key: &Self::Key, _value: &Self::Value) -> bool {
+        false
+    }
+}
diff --git a/foyer-storage/src/slice.rs b/foyer-storage/src/slice.rs
new file mode 100644
index 00000000..7eb7d1df
--- /dev/null
+++ b/foyer-storage/src/slice.rs
@@ -0,0 +1,97 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+/// A mutable bytes slice without lifetime checks.
+///
+/// `slice` does NOT care about buffer allocator, because it never allocates or deallocatoes any buffer.
+#[derive(Debug)]
+pub struct SliceMut {
+    ptr: *mut u8,
+    len: usize,
+}
+
+unsafe impl Send for SliceMut {}
+unsafe impl Sync for SliceMut {}
+
+impl SliceMut {
+    /// # Safety
+    ///
+    /// `buf` MUST outlive `SliceMut`.
+    pub unsafe fn new(buf: &mut [u8]) -> Self {
+        let ptr = buf.as_mut_ptr();
+        let len = buf.len();
+        Self { ptr, len }
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn seal(self) -> Slice {
+        unsafe { Slice::new(self.as_ref()) }
+    }
+}
+
+impl AsRef<[u8]> for SliceMut {
+    fn as_ref(&self) -> &[u8] {
+        unsafe { core::slice::from_raw_parts(self.ptr, self.len) }
+    }
+}
+
+impl AsMut<[u8]> for SliceMut {
+    fn as_mut(&mut self) -> &mut [u8] {
+        unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) }
+    }
+}
+
+/// An immutable bytes slice without lifetime checks.
+///
+/// `slice` does NOT care about buffer allocator, because it never allocates or deallocatoes any buffer.
+#[derive(Debug)]
+pub struct Slice {
+    ptr: *const u8,
+    len: usize,
+}
+
+unsafe impl Send for Slice {}
+unsafe impl Sync for Slice {}
+
+impl Slice {
+    /// # Safety
+    ///
+    /// `buf` MUST outlive `Slice`.
+    pub unsafe fn new(buf: &[u8]) -> Self {
+        let ptr = buf.as_ptr();
+        let len = buf.len();
+        Self { ptr, len }
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+impl AsRef<[u8]> for Slice {
+    fn as_ref(&self) -> &[u8] {
+        unsafe { core::slice::from_raw_parts(self.ptr, self.len) }
+    }
+}
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
new file mode 100644
index 00000000..7b13117a
--- /dev/null
+++ b/foyer-storage/src/store.rs
@@ -0,0 +1,274 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, marker::PhantomData, sync::Arc};
+
+use bytes::{Buf, BufMut};
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use foyer_utils::{bits::align_up, queue::AsyncQueue};
+use twox_hash::XxHash64;
+
+use crate::{
+    admission::AdmissionPolicy,
+    device::{BufferAllocator, Device},
+    error::{Error, Result},
+    flusher::Flusher,
+    indices::{Index, Indices},
+    reclaimer::Reclaimer,
+    region::RegionId,
+    region_manager::{RegionEpItemAdapter, RegionManager},
+    reinsertion::ReinsertionPolicy,
+};
+use foyer_common::{Key, Value};
+use std::hash::Hasher;
+
+pub struct StoreConfig<D, AP, EP, RP, EL>
+where
+    D: Device,
+    AP: AdmissionPolicy,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    RP: ReinsertionPolicy,
+    EL: Link,
+{
+    pub eviction_config: EP::Config,
+    pub device_config: D::Config,
+    pub admission: AP,
+    pub reinsertion: RP,
+    pub buffer_pool_size: usize,
+    pub flushers: usize,
+    pub reclaimers: usize,
+}
+
+impl<D, AP, EP, RP, EL> Debug for StoreConfig<D, AP, EP, RP, EL>
+where
+    D: Device,
+    AP: AdmissionPolicy,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    RP: ReinsertionPolicy,
+    EL: Link,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("StoreConfig")
+            .field("eviction_config", &self.eviction_config)
+            .field("device_config", &self.device_config)
+            .field("admission", &self.admission)
+            .field("reinsertion", &self.reinsertion)
+            .field("buffer_pool_size", &self.buffer_pool_size)
+            .field("flushers", &self.flushers)
+            .field("reclaimers", &self.reclaimers)
+            .finish()
+    }
+}
+
+pub struct Store<K, V, BA, D, EP, AP, RP, EL>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    AP: AdmissionPolicy<Key = K, Value = V>,
+    RP: ReinsertionPolicy<Key = K, Value = V>,
+    EL: Link,
+{
+    indices: Arc<Indices<K>>,
+
+    region_manager: Arc<RegionManager<BA, D, EP, EL>>,
+
+    device: D,
+
+    admission: AP,
+
+    _marker: PhantomData<(V, RP)>,
+}
+
+impl<K, V, BA, D, EP, AP, RP, EL> Store<K, V, BA, D, EP, AP, RP, EL>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    AP: AdmissionPolicy<Key = K, Value = V>,
+    RP: ReinsertionPolicy<Key = K, Value = V>,
+    EL: Link,
+{
+    pub async fn open(config: StoreConfig<D, AP, EP, RP, EL>) -> Result<Arc<Self>> {
+        let device = D::open(config.device_config).await?;
+
+        let buffers = Arc::new(AsyncQueue::new());
+        for _ in 0..(config.buffer_pool_size / device.region_size()) {
+            let len = device.region_size();
+            let buffer = device.io_buffer(len, len);
+            buffers.release(buffer);
+        }
+
+        let clean_regions = Arc::new(AsyncQueue::new());
+        for region_id in 0..device.regions() as RegionId {
+            clean_regions.release(region_id);
+        }
+
+        let mut flusher = Arc::new(Flusher::new(config.flushers));
+        let mut reclaimer = Arc::new(Reclaimer::new(config.reclaimers));
+
+        let region_manager = Arc::new(RegionManager::new(
+            device.regions(),
+            config.eviction_config,
+            buffers.clone(),
+            clean_regions.clone(),
+            device.clone(),
+            flusher.clone(),
+            reclaimer.clone(),
+        ));
+
+        let indices = Arc::new(Indices::new(device.regions()));
+
+        let store = Arc::new(Self {
+            indices: indices.clone(),
+            region_manager: region_manager.clone(),
+            device,
+            admission: config.admission,
+            _marker: PhantomData,
+        });
+
+        // TODO(MrCroxx): refine this!!!
+        unsafe {
+            Arc::get_mut_unchecked(&mut flusher).run(buffers, region_manager.clone());
+            Arc::get_mut_unchecked(&mut reclaimer).run(
+                store.clone(),
+                region_manager,
+                clean_regions,
+                config.reinsertion,
+                indices,
+            );
+        }
+        Ok(store)
+    }
+
+    pub async fn insert(&self, key: K, value: V) -> Result<bool> {
+        if !self.admission.judge(&key, &value) {
+            return Ok(false);
+        }
+
+        let serialized_len = self.serialized_len(&key, &value);
+
+        let mut slice = self.region_manager.allocate(serialized_len).await;
+
+        let mut offset = 0;
+        value.write(&mut slice.as_mut()[offset..offset + value.serialized_len()]);
+        offset += value.serialized_len();
+        key.write(&mut slice.as_mut()[offset..offset + key.serialized_len()]);
+        offset += key.serialized_len();
+
+        let checksum = checksum(&slice.as_ref()[..offset]);
+
+        let footer = Footer {
+            key_len: key.serialized_len() as u32,
+            value_len: value.serialized_len() as u32,
+            checksum,
+        };
+        offset = slice.len() - Footer::serialized_len();
+        footer.write(&mut slice.as_mut()[offset..]);
+
+        let index = Index {
+            region: slice.region_id(),
+            version: slice.version(),
+            offset: slice.offset() as u32,
+            len: slice.len() as u32,
+            key_len: key.serialized_len() as u32,
+            value_len: value.serialized_len() as u32,
+
+            key,
+        };
+
+        drop(slice);
+
+        self.indices.insert(index);
+
+        Ok(true)
+    }
+
+    pub async fn lookup(&self, key: &K) -> Result<Option<V>> {
+        let index = match self.indices.lookup(key) {
+            Some(index) => index,
+            None => return Ok(None),
+        };
+
+        self.region_manager.record_access(&index.region).await;
+        let region = self.region_manager.region(&index.region);
+        let start = index.offset as usize;
+        // TODO(MrCroxx): read value and checksum only
+        let end = start + index.len as usize;
+        let slice = match region.load(start..end, index.version).await? {
+            Some(slice) => slice,
+            None => return Ok(None),
+        };
+
+        let checksum = checksum(&slice.as_ref()[..(index.value_len + index.key_len) as usize]);
+        let expected = (&slice.as_ref()[slice.len() - 8..]).get_u64();
+        if checksum != expected {
+            return Err(Error::ChecksumMismatch { checksum, expected });
+        }
+
+        let value = V::read(&slice.as_ref()[..index.value_len as usize]);
+
+        Ok(Some(value))
+    }
+
+    pub fn remove(&self, key: &K) {
+        self.indices.remove(key);
+    }
+
+    fn serialized_len(&self, key: &K, value: &V) -> usize {
+        let unaligned = key.serialized_len() + value.serialized_len() + Footer::serialized_len();
+        align_up(self.device.align(), unaligned)
+    }
+}
+
+struct Footer {
+    key_len: u32,
+    value_len: u32,
+    checksum: u64,
+}
+
+impl Footer {
+    fn serialized_len() -> usize {
+        4 + 4 + 8
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        buf.put_u32(self.key_len);
+        buf.put_u32(self.value_len);
+        buf.put_u64(self.checksum);
+    }
+
+    #[allow(dead_code)]
+    fn read(mut buf: &[u8]) -> Self {
+        let key_len = buf.get_u32();
+        let value_len = buf.get_u32();
+        let checksum = buf.get_u64();
+
+        Self {
+            key_len,
+            value_len,
+            checksum,
+        }
+    }
+}
+
+fn checksum(buf: &[u8]) -> u64 {
+    let mut hasher = XxHash64::with_seed(0);
+    hasher.write(buf);
+    hasher.finish()
+}
diff --git a/foyer-utils/src/bits.rs b/foyer-utils/src/bits.rs
index 8db5e622..7a6e24f6 100644
--- a/foyer-utils/src/bits.rs
+++ b/foyer-utils/src/bits.rs
@@ -26,8 +26,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::fmt::{Debug, Display};
-use std::ops::{Add, BitAnd, Not, Sub};
+use std::{
+    fmt::{Debug, Display},
+    ops::{Add, BitAnd, Not, Sub},
+};
 
 pub trait UnsignedTrait = Add<Output = Self>
     + Sub<Output = Self>
diff --git a/foyer-utils/src/dlist.rs b/foyer-utils/src/dlist.rs
index 7f7a9d44..7143c27f 100644
--- a/foyer-utils/src/dlist.rs
+++ b/foyer-utils/src/dlist.rs
@@ -12,8 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::marker::PhantomData;
-use std::ptr::NonNull;
+use std::{marker::PhantomData, ptr::NonNull};
 
 /// An entry in an intrusive double linked list
 #[derive(Default, Debug)]
diff --git a/foyer-utils/src/queue.rs b/foyer-utils/src/queue.rs
index d2e0a2ae..832d6856 100644
--- a/foyer-utils/src/queue.rs
+++ b/foyer-utils/src/queue.rs
@@ -12,19 +12,22 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::atomic::Ordering;
-use std::sync::Arc;
-use std::{fmt::Debug, sync::atomic::AtomicUsize};
+use std::{
+    fmt::Debug,
+    sync::atomic::{AtomicUsize, Ordering},
+};
 
-use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
-use tokio::sync::Mutex;
+use tokio::sync::{
+    mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
+    Mutex,
+};
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct AsyncQueue<T: Debug> {
     tx: UnboundedSender<T>,
-    rx: Arc<Mutex<UnboundedReceiver<T>>>,
+    rx: Mutex<UnboundedReceiver<T>>,
 
-    size: Arc<AtomicUsize>,
+    size: AtomicUsize,
 }
 
 impl<T: Debug> Default for AsyncQueue<T> {
@@ -38,8 +41,8 @@ impl<T: Debug> AsyncQueue<T> {
         let (tx, rx) = unbounded_channel();
         Self {
             tx,
-            rx: Arc::new(Mutex::new(rx)),
-            size: Arc::new(AtomicUsize::new(0)),
+            rx: Mutex::new(rx),
+            size: AtomicUsize::new(0),
         }
     }
 
@@ -51,6 +54,7 @@ impl<T: Debug> AsyncQueue<T> {
     }
 
     pub fn release(&self, item: T) {
+        self.size.fetch_add(1, Ordering::Relaxed);
         self.tx.send(item).unwrap();
     }
 
diff --git a/foyer/src/container.rs b/foyer/src/container.rs
index 7f13880d..888654fa 100644
--- a/foyer/src/container.rs
+++ b/foyer/src/container.rs
@@ -12,19 +12,16 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::collections::BTreeMap;
-use std::hash::Hasher;
+use std::{collections::BTreeMap, hash::Hasher};
 
-use std::ptr::NonNull;
-use std::sync::Arc;
+use std::{ptr::NonNull, sync::Arc};
 
 use futures::future::try_join_all;
 use itertools::Itertools;
 use tokio::sync::{Mutex, MutexGuard};
 use twox_hash::XxHash64;
 
-use crate::store::Store;
-use crate::{Data, Index, Metrics, WrappedNonNull};
+use crate::{store::Store, Data, Index, Metrics, WrappedNonNull};
 use foyer_policy::eviction::{Handle, Policy};
 
 // TODO(MrCroxx): wrap own result type
@@ -310,10 +307,11 @@ where
 mod tests {
     use super::*;
 
-    use crate::store::tests::MemoryStore;
-    use crate::tests::is_send_sync_static;
-    use foyer_policy::eviction::lru::{Config as LruConfig, Handle as LruHandle, Lru};
-    use foyer_policy::eviction::tinylfu::Handle as TinyLfuHandle;
+    use crate::{store::tests::MemoryStore, tests::is_send_sync_static};
+    use foyer_policy::eviction::{
+        lru::{Config as LruConfig, Handle as LruHandle, Lru},
+        tinylfu::Handle as TinyLfuHandle,
+    };
 
     #[tokio::test]
     async fn test_container_simple() {
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 5637c07e..96ba81c0 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -119,8 +119,7 @@ pub type LruReadOnlyFileStoreCacheConfig<I, D> = container::Config<
 
 pub use metrics::Metrics;
 
-pub use foyer_policy::eviction::lru::Config as LruConfig;
-pub use foyer_policy::eviction::tinylfu::Config as TinyLfuConfig;
+pub use foyer_policy::eviction::{lru::Config as LruConfig, tinylfu::Config as TinyLfuConfig};
 pub use store::read_only_file_store::Config as ReadOnlyFileStoreConfig;
 
 #[cfg(test)]
diff --git a/foyer/src/store/file.rs b/foyer/src/store/file.rs
index bc9bdc15..d249b523 100644
--- a/foyer/src/store/file.rs
+++ b/foyer/src/store/file.rs
@@ -13,16 +13,19 @@
 //  limitations under the License.
 
 use bytes::{Buf, BufMut};
-use nix::sys::stat::fstat;
-use nix::sys::uio::{pread, pwrite};
-
-use super::asyncify;
-use super::error::Result;
-
-use std::fs::{remove_file, File, OpenOptions};
-use std::os::fd::{AsRawFd, RawFd};
-use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicUsize, Ordering};
+use nix::sys::{
+    stat::fstat,
+    uio::{pread, pwrite},
+};
+
+use super::{asyncify, error::Result};
+
+use std::{
+    fs::{remove_file, File, OpenOptions},
+    os::fd::{AsRawFd, RawFd},
+    path::{Path, PathBuf},
+    sync::atomic::{AtomicUsize, Ordering},
+};
 
 #[derive(Clone, Copy, Debug)]
 pub struct Location {
diff --git a/foyer/src/store/mod.rs b/foyer/src/store/mod.rs
index 69a861f7..079e73ce 100644
--- a/foyer/src/store/mod.rs
+++ b/foyer/src/store/mod.rs
@@ -53,8 +53,7 @@ where
 
 #[cfg(test)]
 pub mod tests {
-    use std::collections::HashMap;
-    use std::sync::Arc;
+    use std::{collections::HashMap, sync::Arc};
 
     use parking_lot::RwLock;
 
diff --git a/foyer/src/store/read_only_file_store.rs b/foyer/src/store/read_only_file_store.rs
index 452a1130..0f113468 100644
--- a/foyer/src/store/read_only_file_store.rs
+++ b/foyer/src/store/read_only_file_store.rs
@@ -12,14 +12,21 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::fs::{create_dir_all, read_dir};
-use std::mem::swap;
-use std::path::{Path, PathBuf};
-
-use std::str::pattern::Pattern;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::Arc;
-use std::{collections::HashMap, marker::PhantomData};
+use std::{
+    fs::{create_dir_all, read_dir},
+    mem::swap,
+    path::{Path, PathBuf},
+};
+
+use std::{
+    collections::HashMap,
+    marker::PhantomData,
+    str::pattern::Pattern,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
 
 use async_trait::async_trait;
 
@@ -30,9 +37,12 @@ use tokio::sync::{RwLock, RwLockWriteGuard};
 
 use crate::{Data, Index, Metrics};
 
-use super::error::Result;
-use super::file::{AppendableFile, Location, ReadableFile, WritableFile};
-use super::{asyncify, Store};
+use super::{
+    asyncify,
+    error::Result,
+    file::{AppendableFile, Location, ReadableFile, WritableFile},
+    Store,
+};
 
 pub type FileId = u32;
 pub type SlotId = u32;
diff --git a/rustfmt.toml b/rustfmt.toml
new file mode 100644
index 00000000..d9ba5fdb
--- /dev/null
+++ b/rustfmt.toml
@@ -0,0 +1 @@
+imports_granularity = "Crate"
\ No newline at end of file

From 64b72b4efbbc9b68d4f81992b5378d9e3e9c5e46 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 30 Jun 2023 14:15:12 +0800
Subject: [PATCH 021/261] chore: remove unused deps (#28)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock                 | 63 --------------------------------------
 foyer-common/Cargo.toml    | 32 -------------------
 foyer-intrusive/Cargo.toml | 24 +--------------
 foyer-policy/Cargo.toml    | 12 +-------
 foyer-storage/Cargo.toml   |  2 --
 foyer-utils/Cargo.toml     | 15 ---------
 6 files changed, 2 insertions(+), 146 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ed4a9c62..78ab0aad 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -361,58 +361,21 @@ dependencies = [
 name = "foyer-common"
 version = "0.1.0"
 dependencies = [
- "async-trait",
  "bytes",
- "bytesize",
- "clap",
- "cmsketch",
- "crossbeam",
- "foyer-policy",
- "foyer-utils",
- "futures",
- "hdrhistogram",
- "itertools",
- "libc",
- "memoffset 0.8.0",
- "nix",
- "parking_lot",
- "paste",
- "prometheus",
- "rand",
- "rand_mt",
- "tempfile",
- "thiserror",
- "tokio",
- "tracing",
- "twox-hash",
 ]
 
 [[package]]
 name = "foyer-intrusive"
 version = "0.1.0"
 dependencies = [
- "async-trait",
  "bytes",
- "bytesize",
- "clap",
  "cmsketch",
- "crossbeam",
  "foyer-common",
- "foyer-utils",
- "futures",
- "hdrhistogram",
  "itertools",
- "libc",
  "memoffset 0.8.0",
- "nix",
  "parking_lot",
  "paste",
- "prometheus",
- "rand",
- "rand_mt",
- "tempfile",
  "thiserror",
- "tokio",
  "tracing",
  "twox-hash",
 ]
@@ -421,25 +384,15 @@ dependencies = [
 name = "foyer-policy"
 version = "0.1.0"
 dependencies = [
- "async-trait",
  "bytes",
- "bytesize",
- "clap",
  "cmsketch",
- "crossbeam",
  "foyer-utils",
- "futures",
- "hdrhistogram",
  "itertools",
- "libc",
  "memoffset 0.8.0",
- "nix",
  "parking_lot",
  "paste",
  "prometheus",
  "rand",
- "rand_mt",
- "tempfile",
  "thiserror",
  "tokio",
  "tracing",
@@ -456,14 +409,12 @@ dependencies = [
  "bytesize",
  "clap",
  "cmsketch",
- "crossbeam",
  "foyer-common",
  "foyer-intrusive",
  "foyer-utils",
  "futures",
  "hdrhistogram",
  "itertools",
- "lazy_static",
  "libc",
  "memoffset 0.8.0",
  "nix",
@@ -505,27 +456,13 @@ dependencies = [
 name = "foyer-utils"
 version = "0.1.0"
 dependencies = [
- "async-trait",
  "bytes",
- "bytesize",
- "clap",
- "cmsketch",
- "crossbeam",
- "futures",
- "hdrhistogram",
  "itertools",
- "libc",
  "memoffset 0.8.0",
- "nix",
  "parking_lot",
  "paste",
- "prometheus",
- "rand",
- "rand_mt",
- "tempfile",
  "thiserror",
  "tokio",
- "twox-hash",
 ]
 
 [[package]]
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 6fc49d79..ddadc3dc 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -8,36 +8,4 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1"
 bytes = "1"
-cmsketch = "0.1"
-crossbeam = "0.8"
-foyer-policy = { path = "../foyer-policy" }
-foyer-utils = { path = "../foyer-utils" }
-futures = "0.3"
-itertools = "0.10.5"
-libc = "0.2"
-memoffset = "0.8"
-nix = { version = "0.26", features = ["fs", "mman"] }
-parking_lot = "0.12"
-paste = "1.0"
-prometheus = "0.13"
-rand = "0.8.5"
-thiserror = "1"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
-tracing = "0.1"
-twox-hash = "1"
-
-[dev-dependencies]
-bytesize = "1"
-clap = { version = "4", features = ["derive"] }
-hdrhistogram = "7"
-rand_mt = "4.2.1"
-tempfile = "3"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 9373da11..c13664a5 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -8,36 +8,14 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1"
+
 bytes = "1"
 cmsketch = "0.1"
-crossbeam = "0.8"
 foyer-common = { path = "../foyer-common" }
-foyer-utils = { path = "../foyer-utils" }
-futures = "0.3"
 itertools = "0.10.5"
-libc = "0.2"
 memoffset = "0.8"
-nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
-prometheus = "0.13"
 thiserror = "1"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
 tracing = "0.1"
 twox-hash = "1"
-
-[dev-dependencies]
-bytesize = "1"
-clap = { version = "4", features = ["derive"] }
-hdrhistogram = "7"
-rand = "0.8.5"
-rand_mt = "4.2.1"
-tempfile = "3"
diff --git a/foyer-policy/Cargo.toml b/foyer-policy/Cargo.toml
index 3c46e3af..b72ca73b 100644
--- a/foyer-policy/Cargo.toml
+++ b/foyer-policy/Cargo.toml
@@ -8,16 +8,11 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1"
 bytes = "1"
 cmsketch = "0.1"
-crossbeam = "0.8"
 foyer-utils = { path = "../foyer-utils" }
-futures = "0.3"
 itertools = "0.10.5"
-libc = "0.2"
 memoffset = "0.8"
-nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
 prometheus = "0.13"
@@ -34,9 +29,4 @@ tokio = { version = "1", features = [
 tracing = "0.1"
 twox-hash = "1"
 
-[dev-dependencies]
-bytesize = "1"
-clap = { version = "4", features = ["derive"] }
-hdrhistogram = "7"
-rand_mt = "4.2.1"
-tempfile = "3"
+
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 0a4093ca..a6be9300 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -12,13 +12,11 @@ async-trait = "0.1"
 bitflags = "2.3.1"
 bytes = "1"
 cmsketch = "0.1"
-crossbeam = "0.8"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-utils = { path = "../foyer-utils" }
 futures = "0.3"
 itertools = "0.10.5"
-lazy_static = "1.4.0"
 libc = "0.2"
 memoffset = "0.8"
 nix = { version = "0.26", features = ["fs", "mman"] }
diff --git a/foyer-utils/Cargo.toml b/foyer-utils/Cargo.toml
index e1bfd5b7..f457935d 100644
--- a/foyer-utils/Cargo.toml
+++ b/foyer-utils/Cargo.toml
@@ -8,18 +8,11 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1"
 bytes = "1"
-cmsketch = "0.1"
-crossbeam = "0.8"
-futures = "0.3"
 itertools = "0.10.5"
-libc = "0.2"
 memoffset = "0.8"
-nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
-prometheus = "0.13"
 thiserror = "1"
 tokio = { version = "1", features = [
     "rt",
@@ -29,12 +22,4 @@ tokio = { version = "1", features = [
     "time",
     "signal",
 ] }
-twox-hash = "1"
 
-[dev-dependencies]
-bytesize = "1"
-clap = { version = "4", features = ["derive"] }
-hdrhistogram = "7"
-rand = "0.8.5"
-rand_mt = "4.2.1"
-tempfile = "3"

From 8388de64ba7d18bb6f4ca8c3d519989ad35f6d20 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 30 Jun 2023 15:40:26 +0800
Subject: [PATCH 022/261] feat: enable direct i/o on linux target (#29)

* feat: enable direct i/o on linux target

- enable direct i/o on linux target
- refine flusher and reclaimer

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix unit test

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/eviction/mod.rs |  4 ++
 foyer-storage-bench/src/main.rs     | 31 ++++--------
 foyer-storage/src/device/fs.rs      | 16 ++++--
 foyer-storage/src/error.rs          |  6 +++
 foyer-storage/src/flusher.rs        | 74 +++++++++++++++++++---------
 foyer-storage/src/lib.rs            | 28 +++++++++++
 foyer-storage/src/reclaimer.rs      | 75 ++++++++++++++++++++---------
 foyer-storage/src/region.rs         | 29 ++++++-----
 foyer-storage/src/region_manager.rs |  9 ++--
 foyer-storage/src/store.rs          | 16 +++---
 10 files changed, 194 insertions(+), 94 deletions(-)

diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index bd1c8b07..df04b217 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -55,6 +55,10 @@ where
         };
         ptr.map(|ptr| self.remove(&ptr))
     }
+
+    fn peek(&self) -> Option<&<A::PointerOps as PointerOps>::Pointer> {
+        self.iter().next()
+    }
 }
 
 pub mod lfu;
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index d2689989..ecf0ca45 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -31,16 +31,10 @@ use std::{
 use analyze::{analyze, monitor, Metrics};
 use clap::Parser;
 
-use foyer_intrusive::eviction::lfu::{Lfu, LfuConfig, LfuLink};
+use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
-    admission::AdmitAll,
-    device::{
-        fs::{FsDevice, FsDeviceConfig},
-        io_buffer::AlignedAllocator,
-    },
-    region_manager::RegionEpItemAdapter,
-    reinsertion::ReinsertNone,
-    store::{Store, StoreConfig},
+    admission::AdmitAll, device::fs::FsDeviceConfig, reinsertion::ReinsertNone, store::StoreConfig,
+    LfuFsStore,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -120,16 +114,7 @@ impl Args {
     }
 }
 
-type TStore = Store<
-    u64,
-    Vec<u8>,
-    AlignedAllocator,
-    FsDevice,
-    Lfu<RegionEpItemAdapter<LfuLink>>,
-    AdmitAll<u64, Vec<u8>>,
-    ReinsertNone<u64, Vec<u8>>,
-    LfuLink,
->;
+type TStore = LfuFsStore<u64, Vec<u8>, AdmitAll<u64, Vec<u8>>, ReinsertNone<u64, Vec<u8>>>;
 
 fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
@@ -310,7 +295,7 @@ async fn write(
 
         let idx = index.fetch_add(1, Ordering::Relaxed);
         // TODO(MrCroxx): Use random content?
-        let data = vec![b'x'; entry_size];
+        let data = vec![idx as u8; entry_size];
         if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
             tokio::time::sleep(wait).await;
         }
@@ -363,9 +348,11 @@ async fn read(
         }
 
         let time = Instant::now();
-        let hit = store.lookup(&idx).await.unwrap().is_some();
+        let res = store.lookup(&idx).await.unwrap();
         let lat = time.elapsed().as_micros() as u64;
-        if hit {
+
+        if res.is_some() {
+            assert_eq!(vec![idx as u8; entry_size], res.unwrap());
             metrics
                 .get_hit_lats
                 .write()
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 80ace582..fb54b40e 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -14,7 +14,10 @@
 
 use std::{
     fs::{create_dir_all, File, OpenOptions},
-    os::fd::{AsRawFd, RawFd},
+    os::{
+        fd::{AsRawFd, RawFd},
+        unix::prelude::OpenOptionsExt,
+    },
     path::PathBuf,
     sync::Arc,
 };
@@ -191,8 +194,7 @@ impl FsDevice {
                     opts.create(true);
                     opts.write(true);
                     opts.read(true);
-
-                    // TODO(MrCroxx): use direct i/o on linux targets
+                    opts.custom_flags(libc::O_DIRECT);
 
                     let file = opts.open(path).map_err(Error::io)?;
 
@@ -228,6 +230,8 @@ impl FsDevice {
 #[cfg(test)]
 mod tests {
 
+    use bytes::BufMut;
+
     use crate::slice::{Slice, SliceMut};
 
     use super::*;
@@ -249,8 +253,10 @@ mod tests {
         };
         let dev = FsDevice::open(config).await.unwrap();
 
-        let wbuffer = vec![b'x'; ALIGN];
-        let mut rbuffer = vec![0; ALIGN];
+        let mut wbuffer = dev.io_buffer(ALIGN, ALIGN);
+        (&mut wbuffer[..]).put_slice(&[b'x'; ALIGN]);
+        let mut rbuffer = dev.io_buffer(ALIGN, ALIGN);
+        (&mut rbuffer[..]).put_slice(&[0; ALIGN]);
 
         let wbuf = unsafe { Slice::new(&wbuffer) };
         let rbuf = unsafe { SliceMut::new(&mut rbuffer) };
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index 7ebc812b..7a0b0f47 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -20,6 +20,8 @@ pub enum Error {
     EntryTooLarge { len: usize, capacity: usize },
     #[error("checksum mismatch, checksum: {checksum}, expected: {expected}")]
     ChecksumMismatch { checksum: u64, expected: u64 },
+    #[error("channel full")]
+    ChannelFull,
     #[error("other error: {0}")]
     Other(String),
 }
@@ -28,6 +30,10 @@ impl Error {
     pub fn device(e: super::device::error::Error) -> Self {
         Self::Device(e)
     }
+
+    pub fn other(e: impl Into<Box<dyn std::error::Error>>) -> Self {
+        Self::Other(e.into().to_string())
+    }
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 09d1a8ed..33d791cd 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -12,19 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::{
-    atomic::{AtomicUsize, Ordering},
-    Arc,
-};
+use std::sync::Arc;
 
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use foyer_utils::queue::AsyncQueue;
 use itertools::Itertools;
-use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+use tokio::sync::{
+    mpsc::{channel, error::TrySendError, Receiver, Sender},
+    Mutex,
+};
 
 use crate::{
     device::{BufferAllocator, Device},
-    error::Result,
+    error::{Error, Result},
     region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
     slice::Slice,
@@ -35,22 +35,32 @@ pub struct FlushTask {
     pub region_id: RegionId,
 }
 
+struct FlusherInner {
+    sequence: usize,
+
+    task_txs: Vec<Sender<FlushTask>>,
+}
+
 pub struct Flusher {
-    sequence: AtomicUsize,
+    runners: usize,
 
-    task_txs: Vec<UnboundedSender<FlushTask>>,
+    inner: Mutex<FlusherInner>,
 }
 
 impl Flusher {
     pub fn new(runners: usize) -> Self {
-        Self {
-            sequence: AtomicUsize::new(0),
+        let inner = FlusherInner {
+            sequence: 0,
             task_txs: Vec::with_capacity(runners),
+        };
+        Self {
+            runners,
+            inner: Mutex::new(inner),
         }
     }
 
-    pub fn run<A, D, E, EL>(
-        &mut self,
+    pub async fn run<A, D, E, EL>(
+        &self,
         buffers: Arc<AsyncQueue<Vec<u8, A>>>,
         region_manager: Arc<RegionManager<A, D, E, EL>>,
     ) where
@@ -59,14 +69,12 @@ impl Flusher {
         E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
         EL: Link,
     {
-        let runners = self.task_txs.capacity();
+        let mut inner = self.inner.lock().await;
 
         #[allow(clippy::type_complexity)]
-        let (mut txs, rxs): (
-            Vec<UnboundedSender<FlushTask>>,
-            Vec<UnboundedReceiver<FlushTask>>,
-        ) = (0..runners).map(|_| unbounded_channel()).unzip();
-        self.task_txs.append(&mut txs);
+        let (mut txs, rxs): (Vec<Sender<FlushTask>>, Vec<Receiver<FlushTask>>) =
+            (0..self.runners).map(|_| channel(1)).unzip();
+        inner.task_txs.append(&mut txs);
 
         let runners = rxs
             .into_iter()
@@ -84,9 +92,31 @@ impl Flusher {
         }
     }
 
-    pub fn submit(&self, task: FlushTask) {
-        let submittee = self.sequence.fetch_add(1, Ordering::Relaxed) % self.task_txs.len();
-        self.task_txs[submittee].send(task).unwrap();
+    pub fn runners(&self) -> usize {
+        self.runners
+    }
+
+    pub async fn submit(&self, task: FlushTask) -> Result<()> {
+        let mut inner = self.inner.lock().await;
+        let submittee = inner.sequence % inner.task_txs.len();
+        inner.sequence += 1;
+        inner.task_txs[submittee]
+            .send(task)
+            .await
+            .map_err(Error::other)
+    }
+
+    pub async fn try_submit(&self, task: FlushTask) -> Result<()> {
+        let mut inner = self.inner.lock().await;
+        let submittee = inner.sequence % inner.task_txs.len();
+        match inner.task_txs[submittee].try_send(task) {
+            Ok(()) => {
+                inner.sequence += 1;
+                Ok(())
+            }
+            Err(TrySendError::Full(_)) => Err(Error::ChannelFull),
+            Err(e) => Err(Error::Other(e.to_string())),
+        }
     }
 }
 
@@ -97,7 +127,7 @@ where
     E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
     EL: Link,
 {
-    task_rx: UnboundedReceiver<FlushTask>,
+    task_rx: Receiver<FlushTask>,
     buffers: Arc<AsyncQueue<Vec<u8, A>>>,
 
     region_manager: Arc<RegionManager<A, D, E, EL>>,
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index d969ee22..ee310788 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -18,6 +18,8 @@
 #![feature(get_mut_unchecked)]
 #![allow(clippy::type_complexity)]
 
+use device::io_buffer::AlignedAllocator;
+
 pub mod admission;
 pub mod device;
 pub mod error;
@@ -29,3 +31,29 @@ pub mod region_manager;
 pub mod reinsertion;
 pub mod slice;
 pub mod store;
+
+pub type LruFsStore<K, V, AP, RP> = store::Store<
+    K,
+    V,
+    AlignedAllocator,
+    device::fs::FsDevice,
+    foyer_intrusive::eviction::lru::Lru<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
+    >,
+    AP,
+    RP,
+    foyer_intrusive::eviction::lru::LruLink,
+>;
+
+pub type LfuFsStore<K, V, AP, RP> = store::Store<
+    K,
+    V,
+    AlignedAllocator,
+    device::fs::FsDevice,
+    foyer_intrusive::eviction::lfu::Lfu<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
+    >,
+    AP,
+    RP,
+    foyer_intrusive::eviction::lfu::LfuLink,
+>;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index af6c9612..876155c1 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -12,15 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::{
-    atomic::{AtomicUsize, Ordering},
-    Arc,
-};
+use std::sync::Arc;
 
 use crate::{
     admission::AdmissionPolicy,
     device::{BufferAllocator, Device},
-    error::Result,
+    error::{Error, Result},
     indices::Indices,
     region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
@@ -31,29 +28,43 @@ use foyer_common::{Key, Value};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use foyer_utils::queue::AsyncQueue;
 use itertools::Itertools;
-use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+use tokio::sync::{
+    mpsc::{channel, error::TrySendError, Receiver, Sender},
+    Mutex,
+};
 
 #[derive(Debug)]
 pub struct ReclaimTask {
     pub region_id: RegionId,
 }
 
+struct ReclaimerInner {
+    sequence: usize,
+
+    task_txs: Vec<Sender<ReclaimTask>>,
+}
+
 pub struct Reclaimer {
-    sequence: AtomicUsize,
+    runners: usize,
 
-    task_txs: Vec<UnboundedSender<ReclaimTask>>,
+    inner: Mutex<ReclaimerInner>,
 }
 
 impl Reclaimer {
     pub fn new(runners: usize) -> Self {
-        Self {
-            sequence: AtomicUsize::new(0),
+        let inner = ReclaimerInner {
+            sequence: 0,
             task_txs: Vec::with_capacity(runners),
+        };
+
+        Self {
+            runners,
+            inner: Mutex::new(inner),
         }
     }
 
-    pub fn run<K, V, A, D, EP, AP, RP, EL>(
-        &mut self,
+    pub async fn run<K, V, A, D, EP, AP, RP, EL>(
+        &self,
         store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
         region_manager: Arc<RegionManager<A, D, EP, EL>>,
         clean_regions: Arc<AsyncQueue<RegionId>>,
@@ -69,14 +80,12 @@ impl Reclaimer {
         RP: ReinsertionPolicy<Key = K, Value = V>,
         EL: Link,
     {
-        let runners = self.task_txs.capacity();
+        let mut inner = self.inner.lock().await;
 
         #[allow(clippy::type_complexity)]
-        let (mut txs, rxs): (
-            Vec<UnboundedSender<ReclaimTask>>,
-            Vec<UnboundedReceiver<ReclaimTask>>,
-        ) = (0..runners).map(|_| unbounded_channel()).unzip();
-        self.task_txs.append(&mut txs);
+        let (mut txs, rxs): (Vec<Sender<ReclaimTask>>, Vec<Receiver<ReclaimTask>>) =
+            (0..self.runners).map(|_| channel(1)).unzip();
+        inner.task_txs.append(&mut txs);
 
         let runners = rxs
             .into_iter()
@@ -97,9 +106,31 @@ impl Reclaimer {
         }
     }
 
-    pub fn submit(&self, task: ReclaimTask) {
-        let submittee = self.sequence.fetch_add(1, Ordering::Relaxed) % self.task_txs.len();
-        self.task_txs[submittee].send(task).unwrap();
+    pub fn runners(&self) -> usize {
+        self.runners
+    }
+
+    pub async fn submit(&self, task: ReclaimTask) -> Result<()> {
+        let mut inner = self.inner.lock().await;
+        let submittee = inner.sequence % inner.task_txs.len();
+        inner.sequence += 1;
+        inner.task_txs[submittee]
+            .send(task)
+            .await
+            .map_err(Error::other)
+    }
+
+    pub async fn try_submit(&self, task: ReclaimTask) -> Result<()> {
+        let mut inner = self.inner.lock().await;
+        let submittee = inner.sequence % inner.task_txs.len();
+        match inner.task_txs[submittee].try_send(task) {
+            Ok(()) => {
+                inner.sequence += 1;
+                Ok(())
+            }
+            Err(TrySendError::Full(_)) => Err(Error::ChannelFull),
+            Err(e) => Err(Error::Other(e.to_string())),
+        }
     }
 }
 
@@ -114,7 +145,7 @@ where
     RP: ReinsertionPolicy<Key = K, Value = V>,
     EL: Link,
 {
-    task_rx: UnboundedReceiver<ReclaimTask>,
+    task_rx: Receiver<ReclaimTask>,
 
     _store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
     region_manager: Arc<RegionManager<A, D, EP, EL>>,
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 6cc4ed32..4bdea8cb 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -188,19 +188,24 @@ where
         }
 
         let region = self.id;
-        let offset = start;
-        let len = end - start;
-
-        tracing::trace!(
-            "read from disk, region: {}, offset: {}, len: {}",
-            region,
-            offset,
-            len
-        );
+        let mut buf = self.device.io_buffer(end - start, end - start);
+
+        let mut offset = 0;
+        while start + offset < end {
+            let len = std::cmp::min(self.device.io_size(), end - start - offset);
+            tracing::trace!(
+                "physical read region {} [{}..{}]",
+                region,
+                start + offset,
+                start + offset + len
+            );
+            let s = unsafe { SliceMut::new(&mut buf[offset..offset + len]) };
+            self.device
+                .read(s, region, (start + offset) as u64, len)
+                .await?;
+            offset += len;
+        }
 
-        let mut buf = self.device.io_buffer(len, len);
-        let slice = unsafe { SliceMut::new(&mut buf) };
-        self.device.read(slice, region, offset as u64, len).await?;
         let callback = {
             let inner = self.inner.clone();
             move || {
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index aed6626c..99eca09d 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -134,16 +134,19 @@ where
             }
 
             // current region is full, schedule flushing
-            self.flusher.submit(FlushTask { region_id });
+            self.flusher.submit(FlushTask { region_id }).await.unwrap();
             inner.current = None;
         }
 
         assert!(inner.current.is_none());
 
         tracing::debug!("clean regions: {}", inner.clean_regions.len());
-        if inner.clean_regions.is_empty() {
+        if inner.clean_regions.len() < self.reclaimer.runners() {
             if let Some(item) = inner.eviction.pop() {
-                self.reclaimer.submit(ReclaimTask { region_id: item.id });
+                self.reclaimer
+                    .submit(ReclaimTask { region_id: item.id })
+                    .await
+                    .unwrap();
             }
         }
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 7b13117a..d0c56872 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -119,8 +119,8 @@ where
             clean_regions.release(region_id);
         }
 
-        let mut flusher = Arc::new(Flusher::new(config.flushers));
-        let mut reclaimer = Arc::new(Reclaimer::new(config.reclaimers));
+        let flusher = Arc::new(Flusher::new(config.flushers));
+        let reclaimer = Arc::new(Reclaimer::new(config.reclaimers));
 
         let region_manager = Arc::new(RegionManager::new(
             device.regions(),
@@ -142,17 +142,17 @@ where
             _marker: PhantomData,
         });
 
-        // TODO(MrCroxx): refine this!!!
-        unsafe {
-            Arc::get_mut_unchecked(&mut flusher).run(buffers, region_manager.clone());
-            Arc::get_mut_unchecked(&mut reclaimer).run(
+        flusher.run(buffers, region_manager.clone()).await;
+        reclaimer
+            .run(
                 store.clone(),
                 region_manager,
                 clean_regions,
                 config.reinsertion,
                 indices,
-            );
-        }
+            )
+            .await;
+
         Ok(store)
     }
 

From 7a5624effc2d1b8f95900028fad51e9fa4d024fb Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 30 Jun 2023 16:18:02 +0800
Subject: [PATCH 023/261] chore: remove unused old storage engien and other
 components (#30)

* chore: remove unused old storage engien and other components

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml              |   9 -
 .github/workflows/main.yml                 |   9 -
 .github/workflows/pull-request.yml         |   9 -
 Cargo.lock                                 |  56 +-
 Cargo.toml                                 |   3 -
 foyer-bench/Cargo.toml                     |  30 -
 foyer-bench/src/analyze.rs                 | 278 ---------
 foyer-bench/src/main.rs                    | 380 ------------
 foyer-bench/src/rate.rs                    |  59 --
 foyer-bench/src/utils.rs                   | 170 ------
 foyer-common/Cargo.toml                    |   2 +
 {foyer-utils => foyer-common}/src/bits.rs  |   0
 foyer-common/src/code.rs                   | 103 ++++
 foyer-common/src/lib.rs                    |  88 +--
 {foyer-utils => foyer-common}/src/queue.rs |   0
 foyer-intrusive/src/core/adapter.rs        |   2 +-
 foyer-policy/Cargo.toml                    |  32 -
 foyer-policy/src/eviction/lru.rs           | 338 -----------
 foyer-policy/src/eviction/mod.rs           |  52 --
 foyer-policy/src/eviction/tinylfu.rs       | 493 ---------------
 foyer-policy/src/lib.rs                    |  30 -
 foyer-policy/src/reinsertion/mod.rs        |  29 -
 foyer-storage/Cargo.toml                   |   1 -
 foyer-storage/src/admission.rs             |   2 +-
 foyer-storage/src/device/io_buffer.rs      |   2 +-
 foyer-storage/src/flusher.rs               |   2 +-
 foyer-storage/src/indices.rs               |   2 +-
 foyer-storage/src/reclaimer.rs             |   6 +-
 foyer-storage/src/region_manager.rs        |   2 +-
 foyer-storage/src/reinsertion.rs           |   2 +-
 foyer-storage/src/store.rs                 |   4 +-
 foyer-utils/Cargo.toml                     |  25 -
 foyer-utils/src/dlist.rs                   | 606 -------------------
 foyer-utils/src/lib.rs                     |  19 -
 foyer/Cargo.toml                           |   2 -
 foyer/src/container.rs                     | 355 -----------
 foyer/src/lib.rs                           | 101 ----
 foyer/src/store/error.rs                   |  25 -
 foyer/src/store/file.rs                    | 403 -------------
 foyer/src/store/mod.rs                     | 106 ----
 foyer/src/store/read_only_file_store.rs    | 671 ---------------------
 41 files changed, 123 insertions(+), 4385 deletions(-)
 delete mode 100644 foyer-bench/Cargo.toml
 delete mode 100644 foyer-bench/src/analyze.rs
 delete mode 100644 foyer-bench/src/main.rs
 delete mode 100644 foyer-bench/src/rate.rs
 delete mode 100644 foyer-bench/src/utils.rs
 rename {foyer-utils => foyer-common}/src/bits.rs (100%)
 create mode 100644 foyer-common/src/code.rs
 rename {foyer-utils => foyer-common}/src/queue.rs (100%)
 delete mode 100644 foyer-policy/Cargo.toml
 delete mode 100644 foyer-policy/src/eviction/lru.rs
 delete mode 100644 foyer-policy/src/eviction/mod.rs
 delete mode 100644 foyer-policy/src/eviction/tinylfu.rs
 delete mode 100644 foyer-policy/src/lib.rs
 delete mode 100644 foyer-policy/src/reinsertion/mod.rs
 delete mode 100644 foyer-utils/Cargo.toml
 delete mode 100644 foyer-utils/src/dlist.rs
 delete mode 100644 foyer-utils/src/lib.rs
 delete mode 100644 foyer/src/container.rs
 delete mode 100644 foyer/src/store/error.rs
 delete mode 100644 foyer/src/store/file.rs
 delete mode 100644 foyer/src/store/mod.rs
 delete mode 100644 foyer/src/store/read_only_file_store.rs

diff --git a/.github/template/template.yml b/.github/template/template.yml
index b6482c4a..fff377d7 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -91,15 +91,6 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
-      - name: Run foyer-bench With Address Sanitizer
-        env:
-          RUST_BACKTRACE: 1
-          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
-        run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench --capacity 256
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0f8cc01a..f00667d2 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -97,15 +97,6 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
-      - name: Run foyer-bench With Address Sanitizer
-        env:
-          RUST_BACKTRACE: 1
-          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
-        run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench --capacity 256
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 626cacbf..bfb037f0 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -96,15 +96,6 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
-      - name: Run foyer-bench With Address Sanitizer
-        env:
-          RUST_BACKTRACE: 1
-          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
-        run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench --capacity 256
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
diff --git a/Cargo.lock b/Cargo.lock
index 78ab0aad..19376d0a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -318,8 +318,6 @@ dependencies = [
  "cmsketch",
  "crossbeam",
  "foyer-common",
- "foyer-policy",
- "foyer-utils",
  "futures",
  "hdrhistogram",
  "itertools",
@@ -338,30 +336,13 @@ dependencies = [
  "twox-hash",
 ]
 
-[[package]]
-name = "foyer-bench"
-version = "0.1.0"
-dependencies = [
- "bytesize",
- "clap",
- "foyer",
- "futures",
- "hdrhistogram",
- "itertools",
- "libc",
- "nix",
- "parking_lot",
- "rand",
- "rand_mt",
- "tempfile",
- "tokio",
-]
-
 [[package]]
 name = "foyer-common"
 version = "0.1.0"
 dependencies = [
  "bytes",
+ "paste",
+ "tokio",
 ]
 
 [[package]]
@@ -380,25 +361,6 @@ dependencies = [
  "twox-hash",
 ]
 
-[[package]]
-name = "foyer-policy"
-version = "0.1.0"
-dependencies = [
- "bytes",
- "cmsketch",
- "foyer-utils",
- "itertools",
- "memoffset 0.8.0",
- "parking_lot",
- "paste",
- "prometheus",
- "rand",
- "thiserror",
- "tokio",
- "tracing",
- "twox-hash",
-]
-
 [[package]]
 name = "foyer-storage"
 version = "0.1.0"
@@ -411,7 +373,6 @@ dependencies = [
  "cmsketch",
  "foyer-common",
  "foyer-intrusive",
- "foyer-utils",
  "futures",
  "hdrhistogram",
  "itertools",
@@ -452,19 +413,6 @@ dependencies = [
  "tracing-subscriber",
 ]
 
-[[package]]
-name = "foyer-utils"
-version = "0.1.0"
-dependencies = [
- "bytes",
- "itertools",
- "memoffset 0.8.0",
- "parking_lot",
- "paste",
- "thiserror",
- "tokio",
-]
-
 [[package]]
 name = "futures"
 version = "0.3.28"
diff --git a/Cargo.toml b/Cargo.toml
index e7ad61ee..59544b15 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,13 +2,10 @@
 
 members = [
     "foyer",
-    "foyer-bench",
     "foyer-common",
     "foyer-intrusive",
-    "foyer-policy",
     "foyer-storage",
     "foyer-storage-bench",
-    "foyer-utils",
 ]
 
 [patch.crates-io]
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
deleted file mode 100644
index 241b1933..00000000
--- a/foyer-bench/Cargo.toml
+++ /dev/null
@@ -1,30 +0,0 @@
-[package]
-name = "foyer-bench"
-version = "0.1.0"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
-license = "Apache-2.0"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-bytesize = "1"
-clap = { version = "4", features = ["derive"] }
-foyer = { path = "../foyer" }
-futures = "0.3"
-hdrhistogram = "7"
-itertools = "0.10.5"
-libc = "0.2"
-nix = { version = "0.26", features = ["fs", "mman"] }
-parking_lot = "0.12"
-rand = "0.8.5"
-rand_mt = "4.2.1"
-tempfile = "3"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
diff --git a/foyer-bench/src/analyze.rs b/foyer-bench/src/analyze.rs
deleted file mode 100644
index 82a852cd..00000000
--- a/foyer-bench/src/analyze.rs
+++ /dev/null
@@ -1,278 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-// Copyright 2023 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::{
-    path::Path,
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc,
-    },
-    time::{Duration, Instant},
-};
-
-use bytesize::ByteSize;
-use hdrhistogram::Histogram;
-use parking_lot::RwLock;
-use tokio::sync::oneshot;
-
-use crate::utils::{iostat, IoStat};
-
-const SECTOR_SIZE: usize = 512;
-
-// latencies are measured by 'us'
-#[derive(Clone, Copy, Debug)]
-pub struct Analysis {
-    disk_read_iops: f64,
-    disk_read_throughput: f64,
-    disk_write_iops: f64,
-    disk_write_throughput: f64,
-
-    insert_iops: f64,
-    insert_throughput: f64,
-    insert_lat_p50: u64,
-    insert_lat_p90: u64,
-    insert_lat_p99: u64,
-
-    get_iops: f64,
-    get_miss: f64,
-    get_miss_lat_p50: u64,
-    get_miss_lat_p90: u64,
-    get_miss_lat_p99: u64,
-    get_hit_lat_p50: u64,
-    get_hit_lat_p90: u64,
-    get_hit_lat_p99: u64,
-}
-
-#[derive(Default, Clone, Copy, Debug)]
-pub struct MetricsDump {
-    pub insert_ios: usize,
-    pub insert_bytes: usize,
-    pub insert_lat_p50: u64,
-    pub insert_lat_p90: u64,
-    pub insert_lat_p99: u64,
-
-    pub get_ios: usize,
-    pub get_miss_ios: usize,
-    pub get_hit_lat_p50: u64,
-    pub get_hit_lat_p90: u64,
-    pub get_hit_lat_p99: u64,
-    pub get_miss_lat_p50: u64,
-    pub get_miss_lat_p90: u64,
-    pub get_miss_lat_p99: u64,
-}
-
-#[derive(Clone, Debug)]
-pub struct Metrics {
-    pub insert_ios: Arc<AtomicUsize>,
-    pub insert_bytes: Arc<AtomicUsize>,
-    pub insert_lats: Arc<RwLock<Histogram<u64>>>,
-
-    pub get_ios: Arc<AtomicUsize>,
-    pub get_miss_ios: Arc<AtomicUsize>,
-    pub get_hit_lats: Arc<RwLock<Histogram<u64>>>,
-    pub get_miss_lats: Arc<RwLock<Histogram<u64>>>,
-}
-
-impl Default for Metrics {
-    fn default() -> Self {
-        Self {
-            insert_ios: Arc::new(AtomicUsize::new(0)),
-            insert_bytes: Arc::new(AtomicUsize::new(0)),
-            insert_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
-
-            get_ios: Arc::new(AtomicUsize::new(0)),
-            get_miss_ios: Arc::new(AtomicUsize::new(0)),
-            get_hit_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
-            get_miss_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
-        }
-    }
-}
-
-impl Metrics {
-    pub fn dump(&self) -> MetricsDump {
-        let insert_lats = self.insert_lats.read();
-        let get_hit_lats = self.get_hit_lats.read();
-        let get_miss_lats = self.get_miss_lats.read();
-        MetricsDump {
-            insert_ios: self.insert_ios.load(Ordering::Relaxed),
-            insert_bytes: self.insert_bytes.load(Ordering::Relaxed),
-            insert_lat_p50: insert_lats.value_at_quantile(0.5),
-            insert_lat_p90: insert_lats.value_at_quantile(0.9),
-            insert_lat_p99: insert_lats.value_at_quantile(0.99),
-
-            get_ios: self.get_ios.load(Ordering::Relaxed),
-            get_miss_ios: self.get_miss_ios.load(Ordering::Relaxed),
-            get_hit_lat_p50: get_hit_lats.value_at_quantile(0.5),
-            get_hit_lat_p90: get_hit_lats.value_at_quantile(0.9),
-            get_hit_lat_p99: get_hit_lats.value_at_quantile(0.99),
-            get_miss_lat_p50: get_miss_lats.value_at_quantile(0.5),
-            get_miss_lat_p90: get_miss_lats.value_at_quantile(0.9),
-            get_miss_lat_p99: get_miss_lats.value_at_quantile(0.99),
-        }
-    }
-}
-
-impl std::fmt::Display for Analysis {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let disk_read_throughput = ByteSize::b(self.disk_read_throughput as u64);
-        let disk_write_throughput = ByteSize::b(self.disk_write_throughput as u64);
-        let disk_total_throughput = disk_read_throughput + disk_write_throughput;
-
-        // disk statics
-        writeln!(
-            f,
-            "disk total iops: {:.1}",
-            self.disk_write_iops + self.disk_read_iops
-        )?;
-        writeln!(
-            f,
-            "disk total throughput: {}/s",
-            disk_total_throughput.to_string_as(true)
-        )?;
-        writeln!(f, "disk read iops: {:.1}", self.disk_read_iops)?;
-        writeln!(
-            f,
-            "disk read throughput: {}/s",
-            disk_read_throughput.to_string_as(true)
-        )?;
-        writeln!(f, "disk write iops: {:.1}", self.disk_write_iops)?;
-        writeln!(
-            f,
-            "disk write throughput: {}/s",
-            disk_write_throughput.to_string_as(true)
-        )?;
-
-        // insert statics
-        let insert_throughput = ByteSize::b(self.insert_throughput as u64);
-        writeln!(f, "insert iops: {:.1}/s", self.insert_iops)?;
-        writeln!(
-            f,
-            "insert throughput: {}/s",
-            insert_throughput.to_string_as(true)
-        )?;
-        writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
-        writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
-        writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
-
-        // get statics
-        writeln!(f, "get iops: {:.1}/s", self.get_iops)?;
-        writeln!(f, "get miss: {:.2}% ", self.get_miss * 100f64)?;
-        writeln!(f, "get hit lat p50: {}us", self.get_hit_lat_p50)?;
-        writeln!(f, "get hit lat p90: {}us", self.get_hit_lat_p90)?;
-        writeln!(f, "get hit lat p99: {}us", self.get_hit_lat_p99)?;
-        writeln!(f, "get miss lat p50: {}us", self.get_miss_lat_p50)?;
-        writeln!(f, "get miss lat p90: {}us", self.get_miss_lat_p90)?;
-        writeln!(f, "get miss lat p99: {}us", self.get_miss_lat_p99)?;
-
-        Ok(())
-    }
-}
-
-pub fn analyze(
-    duration: Duration,
-    iostat_start: &IoStat,
-    iostat_end: &IoStat,
-    metrics_dump_start: &MetricsDump,
-    metrics_dump_end: &MetricsDump,
-) -> Analysis {
-    let secs = duration.as_secs_f64();
-    let disk_read_iops = (iostat_end.read_ios - iostat_start.read_ios) as f64 / secs;
-    let disk_read_throughput =
-        (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
-    let disk_write_iops = (iostat_end.write_ios - iostat_start.write_ios) as f64 / secs;
-    let disk_write_throughput =
-        (iostat_end.write_sectors - iostat_start.write_sectors) as f64 * SECTOR_SIZE as f64 / secs;
-
-    let insert_iops = (metrics_dump_end.insert_ios - metrics_dump_start.insert_ios) as f64 / secs;
-    let insert_throughput =
-        (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
-
-    let get_iops = (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64 / secs;
-    let get_miss = (metrics_dump_end.get_miss_ios - metrics_dump_start.get_miss_ios) as f64
-        / (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64;
-
-    Analysis {
-        disk_read_iops,
-        disk_read_throughput,
-        disk_write_iops,
-        disk_write_throughput,
-
-        insert_iops,
-        insert_throughput,
-        insert_lat_p50: metrics_dump_end.insert_lat_p50,
-        insert_lat_p90: metrics_dump_end.insert_lat_p90,
-        insert_lat_p99: metrics_dump_end.insert_lat_p99,
-
-        get_iops,
-        get_miss,
-        get_hit_lat_p50: metrics_dump_end.get_hit_lat_p50,
-        get_hit_lat_p90: metrics_dump_end.get_hit_lat_p90,
-        get_hit_lat_p99: metrics_dump_end.get_hit_lat_p99,
-        get_miss_lat_p50: metrics_dump_end.get_miss_lat_p50,
-        get_miss_lat_p90: metrics_dump_end.get_miss_lat_p90,
-        get_miss_lat_p99: metrics_dump_end.get_miss_lat_p99,
-    }
-}
-
-pub async fn monitor(
-    iostat_path: impl AsRef<Path>,
-    interval: Duration,
-    metrics: Metrics,
-    mut stop: oneshot::Receiver<()>,
-) {
-    let mut stat = iostat(&iostat_path);
-    let mut metrics_dump = metrics.dump();
-    loop {
-        let start = Instant::now();
-        match stop.try_recv() {
-            Err(oneshot::error::TryRecvError::Empty) => {}
-            _ => return,
-        }
-
-        tokio::time::sleep(interval).await;
-        let new_stat = iostat(&iostat_path);
-        let new_metrics_dump = metrics.dump();
-        let analysis = analyze(
-            // interval may have ~ +7% error
-            start.elapsed(),
-            &stat,
-            &new_stat,
-            &metrics_dump,
-            &new_metrics_dump,
-        );
-        println!("{}", analysis);
-        stat = new_stat;
-        metrics_dump = new_metrics_dump;
-    }
-}
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
deleted file mode 100644
index d3c5d3d6..00000000
--- a/foyer-bench/src/main.rs
+++ /dev/null
@@ -1,380 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#![feature(let_chains)]
-
-mod analyze;
-mod rate;
-mod utils;
-
-use std::{
-    fs::create_dir_all,
-    path::PathBuf,
-    sync::{
-        atomic::{AtomicU64, Ordering},
-        Arc,
-    },
-    time::{Duration, Instant},
-};
-
-use analyze::{analyze, monitor, Metrics};
-use clap::Parser;
-
-use foyer::{
-    ReadOnlyFileStoreConfig, TinyLfuConfig, TinyLfuReadOnlyFileStoreCache,
-    TinyLfuReadOnlyFileStoreCacheConfig,
-};
-use futures::future::join_all;
-use itertools::Itertools;
-use rand::{rngs::StdRng, Rng, SeedableRng};
-
-use rate::RateLimiter;
-use tokio::sync::oneshot;
-use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
-
-#[derive(Parser, Debug, Clone)]
-pub struct Args {
-    /// dir for cache data
-    #[arg(short, long)]
-    dir: String,
-
-    /// (MiB)
-    #[arg(long, default_value_t = 1024)]
-    capacity: usize,
-
-    /// (s)
-    #[arg(short, long, default_value_t = 60)]
-    time: u64,
-
-    #[arg(long, default_value_t = 16)]
-    concurrency: usize,
-
-    /// must be power of 2
-    #[arg(long, default_value_t = 16)]
-    pools: usize,
-
-    /// (s)
-    #[arg(long, default_value_t = 2)]
-    report_interval: u64,
-
-    /// Some filesystem (e.g. btrfs) can span across multiple block devices and it's hard to decide
-    /// which device to moitor. Use this argument to specify which block device to monitor.
-    #[arg(long, default_value = "")]
-    iostat_dev: String,
-
-    #[arg(long, default_value_t = 0.0)]
-    w_rate: f64,
-
-    #[arg(long, default_value_t = 0.0)]
-    r_rate: f64,
-
-    #[arg(long, default_value_t = 64 * 1024)]
-    entry_size: usize,
-
-    #[arg(long, default_value_t = 10000)]
-    look_up_range: u64,
-
-    /// read only file store: max cache file size (MiB)
-    #[arg(long, default_value_t = 16)]
-    rofs_max_file_size: usize,
-
-    /// read only file store: ratio of garbage to trigger reclaim (0 ~ 1)
-    #[arg(long, default_value_t = 0.0)]
-    rofs_trigger_reclaim_garbage_ratio: f64,
-
-    /// read only file store: ratio of size to trigger reclaim (0 ~ 1)
-    #[arg(long, default_value_t = 0.8)]
-    rofs_trigger_reclaim_capacity_ratio: f64,
-
-    /// read only file store: ratio of size to trigger randomly drop (0 ~ 1)
-    #[arg(long, default_value_t = 0.0)]
-    rofs_trigger_random_drop_ratio: f64,
-
-    /// read only file store: ratio of randomly dropped entries (0 ~ 1)
-    #[arg(long, default_value_t = 0.0)]
-    rofs_random_drop_ratio: f64,
-
-    /// read only file store: ratio of size to trigger write stall, every new entry to insert will be dropped (0 ~ 1)
-    #[arg(long, default_value_t = 0.0)]
-    rofs_write_stall_threshold_ratio: f64,
-}
-
-impl Args {
-    fn verify(&self) {
-        assert!(self.pools.is_power_of_two());
-    }
-}
-type TContainer = TinyLfuReadOnlyFileStoreCache<u64, Vec<u8>>;
-
-fn is_send_sync_static<T: Send + Sync + 'static>() {}
-
-#[tokio::main]
-async fn main() {
-    is_send_sync_static::<TContainer>();
-
-    let args = Args::parse();
-    args.verify();
-
-    println!("{:#?}", args);
-
-    create_dir_all(&args.dir).unwrap();
-
-    let iostat_path = match detect_fs_type(&args.dir) {
-        FsType::Tmpfs => panic!("tmpfs is not supported with benches"),
-        FsType::Btrfs => {
-            if args.iostat_dev.is_empty() {
-                panic!("cannot decide which block device to monitor for btrfs, please specify device name with \'--iostat-dev\'")
-            } else {
-                dev_stat_path(&args.iostat_dev)
-            }
-        }
-        _ => file_stat_path(&args.dir),
-    };
-
-    let metrics = Metrics::default();
-
-    let iostat_start = iostat(&iostat_path);
-    let metrics_dump_start = metrics.dump();
-    let time = Instant::now();
-
-    let store_config = ReadOnlyFileStoreConfig {
-        dir: PathBuf::from(&args.dir),
-        capacity: args.capacity * 1024 * 1024 / args.pools,
-        max_file_size: args.rofs_max_file_size * 1024 * 1024,
-        trigger_reclaim_garbage_ratio: args.rofs_trigger_reclaim_garbage_ratio,
-        trigger_reclaim_capacity_ratio: args.rofs_trigger_reclaim_capacity_ratio,
-        trigger_random_drop_ratio: args.rofs_trigger_random_drop_ratio,
-        random_drop_ratio: args.rofs_random_drop_ratio,
-        write_stall_threshold_ratio: args.rofs_write_stall_threshold_ratio,
-    };
-
-    let policy_config = TinyLfuConfig {
-        window_to_cache_size_ratio: 1,
-        tiny_lru_capacity_ratio: 0.01,
-    };
-
-    let config = TinyLfuReadOnlyFileStoreCacheConfig {
-        capacity: args.capacity * 1024 * 1024,
-        pool_count_bits: (args.pools as f64).log2() as usize,
-        policy_config,
-        store_config,
-    };
-
-    println!("{:#?}", config);
-
-    let container = TinyLfuReadOnlyFileStoreCache::open(config).await.unwrap();
-    let container = Arc::new(container);
-
-    let (iostat_stop_tx, iostat_stop_rx) = oneshot::channel();
-    let (bench_stop_txs, bench_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
-        (0..args.concurrency).map(|_| oneshot::channel()).unzip();
-
-    let handle_monitor = tokio::spawn({
-        let iostat_path = iostat_path.clone();
-        let metrics = metrics.clone();
-
-        monitor(
-            iostat_path,
-            Duration::from_secs(args.report_interval),
-            metrics,
-            iostat_stop_rx,
-        )
-    });
-    let handle_signal = tokio::spawn(async move {
-        tokio::signal::ctrl_c().await.unwrap();
-        for bench_stop_tx in bench_stop_txs {
-            bench_stop_tx.send(()).unwrap();
-        }
-        iostat_stop_tx.send(()).unwrap();
-    });
-    let handles = (bench_stop_rxs)
-        .into_iter()
-        .map(|stop| {
-            tokio::spawn(bench(
-                args.clone(),
-                container.clone(),
-                metrics.clone(),
-                stop,
-            ))
-        })
-        .collect_vec();
-
-    for handle in handles {
-        handle.await.unwrap();
-    }
-    handle_monitor.abort();
-    handle_signal.abort();
-
-    let iostat_end = iostat(&iostat_path);
-    let metrics_dump_end = metrics.dump();
-    let analysis = analyze(
-        time.elapsed(),
-        &iostat_start,
-        &iostat_end,
-        &metrics_dump_start,
-        &metrics_dump_end,
-    );
-    println!("\nTotal:\n{}", analysis);
-}
-
-async fn bench(
-    args: Args,
-    container: Arc<TContainer>,
-    metrics: Metrics,
-    stop: oneshot::Receiver<()>,
-) {
-    let w_rate = if args.w_rate == 0.0 {
-        None
-    } else {
-        Some(args.w_rate)
-    };
-    let r_rate = if args.r_rate == 0.0 {
-        None
-    } else {
-        Some(args.r_rate)
-    };
-
-    let index = Arc::new(AtomicU64::new(0));
-
-    let (w_stop_tx, w_stop_rx) = oneshot::channel();
-    let (r_stop_tx, r_stop_rx) = oneshot::channel();
-
-    let handle_w = tokio::spawn(write(
-        args.entry_size,
-        w_rate,
-        index.clone(),
-        container.clone(),
-        args.time,
-        metrics.clone(),
-        w_stop_rx,
-    ));
-    let handle_r = tokio::spawn(read(
-        args.entry_size,
-        r_rate,
-        index.clone(),
-        container.clone(),
-        args.time,
-        metrics.clone(),
-        r_stop_rx,
-        args.look_up_range,
-    ));
-    tokio::spawn(async move {
-        if let Ok(()) = stop.await {
-            let _ = w_stop_tx.send(());
-            let _ = r_stop_tx.send(());
-        }
-    });
-
-    join_all([handle_r, handle_w]).await;
-}
-
-#[allow(clippy::too_many_arguments)]
-async fn write(
-    entry_size: usize,
-    rate: Option<f64>,
-    index: Arc<AtomicU64>,
-    container: Arc<TContainer>,
-    time: u64,
-    metrics: Metrics,
-    mut stop: oneshot::Receiver<()>,
-) {
-    let start = Instant::now();
-
-    let mut limiter = rate.map(RateLimiter::new);
-
-    loop {
-        match stop.try_recv() {
-            Err(oneshot::error::TryRecvError::Empty) => {}
-            _ => return,
-        }
-        if start.elapsed().as_secs() >= time {
-            return;
-        }
-
-        let idx = index.fetch_add(1, Ordering::Relaxed);
-        // TODO(MrCroxx): Use random content?
-        let data = vec![b'x'; entry_size];
-        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
-            tokio::time::sleep(wait).await;
-        }
-
-        let time = Instant::now();
-        container.insert(idx, data).await.unwrap();
-        metrics
-            .insert_lats
-            .write()
-            .record(time.elapsed().as_micros() as u64)
-            .expect("record out of range");
-        metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
-        metrics
-            .insert_bytes
-            .fetch_add(entry_size, Ordering::Relaxed);
-    }
-}
-
-#[allow(clippy::too_many_arguments)]
-async fn read(
-    entry_size: usize,
-    rate: Option<f64>,
-    index: Arc<AtomicU64>,
-    container: Arc<TContainer>,
-    time: u64,
-    metrics: Metrics,
-    mut stop: oneshot::Receiver<()>,
-    look_up_range: u64,
-) {
-    let start = Instant::now();
-
-    let mut limiter = rate.map(RateLimiter::new);
-
-    let mut rng = StdRng::seed_from_u64(0);
-
-    loop {
-        match stop.try_recv() {
-            Err(oneshot::error::TryRecvError::Empty) => {}
-            _ => return,
-        }
-        if start.elapsed().as_secs() >= time {
-            return;
-        }
-
-        let idx_max = index.load(Ordering::Relaxed);
-        let idx = rng.gen_range(std::cmp::max(idx_max, look_up_range) - look_up_range..=idx_max);
-
-        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
-            tokio::time::sleep(wait).await;
-        }
-
-        let time = Instant::now();
-        let hit = container.get(&idx).await.unwrap().is_some();
-        let lat = time.elapsed().as_micros() as u64;
-        if hit {
-            metrics
-                .get_hit_lats
-                .write()
-                .record(lat)
-                .expect("record out of range");
-        } else {
-            metrics
-                .get_miss_lats
-                .write()
-                .record(lat)
-                .expect("record out of range");
-            metrics.get_miss_ios.fetch_add(1, Ordering::Relaxed);
-        }
-        metrics.get_ios.fetch_add(1, Ordering::Relaxed);
-
-        tokio::task::consume_budget().await;
-    }
-}
diff --git a/foyer-bench/src/rate.rs b/foyer-bench/src/rate.rs
deleted file mode 100644
index 82164aab..00000000
--- a/foyer-bench/src/rate.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-// Copyright 2023 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::{Duration, Instant};
-
-pub struct RateLimiter {
-    capacity: f64,
-    quota: f64,
-
-    last: Instant,
-}
-
-impl RateLimiter {
-    pub fn new(capacity: f64) -> Self {
-        Self {
-            capacity,
-            quota: 0.0,
-            last: Instant::now(),
-        }
-    }
-
-    pub fn consume(&mut self, weight: f64) -> Option<Duration> {
-        let now = Instant::now();
-        let refill = now.duration_since(self.last).as_secs_f64() * self.capacity;
-        self.last = now;
-        self.quota = f64::min(self.quota + refill, self.capacity);
-        self.quota -= weight;
-        if self.quota >= 0.0 {
-            return None;
-        }
-        let wait = Duration::from_secs_f64((-self.quota) / self.capacity);
-        Some(wait)
-    }
-}
diff --git a/foyer-bench/src/utils.rs b/foyer-bench/src/utils.rs
deleted file mode 100644
index d815da36..00000000
--- a/foyer-bench/src/utils.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-// Copyright 2023 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::path::{Path, PathBuf};
-
-use itertools::Itertools;
-use nix::{fcntl::readlink, sys::stat::stat};
-
-#[allow(unused)]
-#[derive(PartialEq, Clone, Copy, Debug)]
-pub enum FsType {
-    Xfs,
-    Ext4,
-    Btrfs,
-    Tmpfs,
-    Others,
-}
-
-#[allow(unused)]
-pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
-    #[cfg(target_os = "linux")]
-    {
-        use nix::sys::statfs::{
-            statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC,
-        };
-        let fs_stat = statfs(path.as_ref()).unwrap();
-        match fs_stat.filesystem_type() {
-            XFS_SUPER_MAGIC => FsType::Xfs,
-            EXT4_SUPER_MAGIC => FsType::Ext4,
-            BTRFS_SUPER_MAGIC => FsType::Btrfs,
-            TMPFS_MAGIC => FsType::Tmpfs,
-            _ => FsType::Others,
-        }
-    }
-
-    #[cfg(not(target_os = "linux"))]
-    FsType::Others
-}
-
-/// Given a normal file path, returns the containing block device static file path (of the
-/// partition).
-pub fn file_stat_path(path: impl AsRef<Path>) -> PathBuf {
-    let st_dev = stat(path.as_ref()).unwrap().st_dev;
-
-    let major = unsafe { libc::major(st_dev) };
-    let minor = unsafe { libc::minor(st_dev) };
-
-    let dev = PathBuf::from("/dev/block").join(format!("{}:{}", major, minor));
-
-    let linkname = readlink(&dev).unwrap();
-    let devname = Path::new(linkname.as_os_str()).file_name().unwrap();
-    dev_stat_path(devname.to_str().unwrap())
-}
-
-pub fn dev_stat_path(devname: &str) -> PathBuf {
-    let classpath = Path::new("/sys/class/block").join(devname);
-    let devclass = readlink(&classpath).unwrap();
-
-    let devpath = Path::new(&devclass);
-    Path::new("/sys")
-        .join(devpath.strip_prefix("../..").unwrap())
-        .join("stat")
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct IoStat {
-    /// read I/Os       requests      number of read I/Os processed
-    pub read_ios: usize,
-    /// read merges     requests      number of read I/Os merged with in-queue I/O
-    pub read_merges: usize,
-    /// read sectors    sectors       number of sectors read
-    pub read_sectors: usize,
-    /// read ticks      milliseconds  total wait time for read requests
-    pub read_ticks: usize,
-    /// write I/Os      requests      number of write I/Os processed
-    pub write_ios: usize,
-    /// write merges    requests      number of write I/Os merged with in-queue I/O
-    pub write_merges: usize,
-    /// write sectors   sectors       number of sectors written
-    pub write_sectors: usize,
-    /// write ticks     milliseconds  total wait time for write requests
-    pub write_ticks: usize,
-    /// in_flight       requests      number of I/Os currently in flight
-    pub in_flight: usize,
-    /// io_ticks        milliseconds  total time this block device has been active
-    pub io_ticks: usize,
-    /// time_in_queue   milliseconds  total wait time for all requests
-    pub time_in_queue: usize,
-    /// discard I/Os    requests      number of discard I/Os processed
-    pub discard_ios: usize,
-    /// discard merges  requests      number of discard I/Os merged with in-queue I/O
-    pub discard_merges: usize,
-    /// discard sectors sectors       number of sectors discarded
-    pub discard_sectors: usize,
-    /// discard ticks   milliseconds  total wait time for discard requests
-    pub discard_ticks: usize,
-    /// flush I/Os      requests      number of flush I/Os processed
-    pub flush_ios: usize,
-    /// flush ticks     milliseconds  total wait time for flush requests
-    pub flush_ticks: usize,
-}
-
-/// Given the device static file path and get the iostat.
-pub fn iostat(path: impl AsRef<Path>) -> IoStat {
-    let content = std::fs::read_to_string(path.as_ref()).unwrap();
-    let nums = content.split_ascii_whitespace().collect_vec();
-
-    let read_ios = nums[0].parse().unwrap();
-    let read_merges = nums[1].parse().unwrap();
-    let read_sectors = nums[2].parse().unwrap();
-    let read_ticks = nums[3].parse().unwrap();
-    let write_ios = nums[4].parse().unwrap();
-    let write_merges = nums[5].parse().unwrap();
-    let write_sectors = nums[6].parse().unwrap();
-    let write_ticks = nums[7].parse().unwrap();
-    let in_flight = nums[8].parse().unwrap();
-    let io_ticks = nums[9].parse().unwrap();
-    let time_in_queue = nums[10].parse().unwrap();
-    let discard_ios = nums[11].parse().unwrap();
-    let discard_merges = nums[12].parse().unwrap();
-    let discard_sectors = nums[13].parse().unwrap();
-    let discard_ticks = nums[14].parse().unwrap();
-    let flush_ios = nums[15].parse().unwrap();
-    let flush_ticks = nums[16].parse().unwrap();
-
-    IoStat {
-        read_ios,
-        read_merges,
-        read_sectors,
-        read_ticks,
-        write_ios,
-        write_merges,
-        write_sectors,
-        write_ticks,
-        in_flight,
-        io_ticks,
-        time_in_queue,
-        discard_ios,
-        discard_merges,
-        discard_sectors,
-        discard_ticks,
-        flush_ios,
-        flush_ticks,
-    }
-}
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index ddadc3dc..eafaee68 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -9,3 +9,5 @@ license = "Apache-2.0"
 
 [dependencies]
 bytes = "1"
+paste = "1.0"
+tokio = { version = "1", features = ["sync"] }
diff --git a/foyer-utils/src/bits.rs b/foyer-common/src/bits.rs
similarity index 100%
rename from foyer-utils/src/bits.rs
rename to foyer-common/src/bits.rs
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
new file mode 100644
index 00000000..0c34d919
--- /dev/null
+++ b/foyer-common/src/code.rs
@@ -0,0 +1,103 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use bytes::{Buf, BufMut};
+use paste::paste;
+
+pub trait Key:
+    Sized
+    + Send
+    + Sync
+    + 'static
+    + std::hash::Hash
+    + Eq
+    + PartialEq
+    + Ord
+    + PartialOrd
+    + Clone
+    + std::fmt::Debug
+{
+    fn serialized_len(&self) -> usize {
+        panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
+    }
+
+    fn write(&self, _buf: &mut [u8]) {
+        panic!("Method `write` must be implemented for `Key` if storage is used.")
+    }
+
+    fn read(_buf: &[u8]) -> Self {
+        panic!("Method `read` must be implemented for `Key` if storage is used.")
+    }
+}
+
+pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
+    fn serialized_len(&self) -> usize {
+        panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
+    }
+
+    fn write(&self, _buf: &mut [u8]) {
+        panic!("Method `write` must be implemented for `Value` if storage is used.")
+    }
+
+    fn read(_buf: &[u8]) -> Self {
+        panic!("Method `read` must be implemented for `Value` if storage is used.")
+    }
+}
+
+macro_rules! for_all_primitives {
+    ($macro:ident) => {
+        $macro! {
+            u8, u16, u32, u64,
+            i8, i16, i32, i64,
+        }
+    };
+}
+
+macro_rules! impl_key {
+    ($( $type:ty, )*) => {
+        paste! {
+            $(
+                impl Key for $type {
+                    fn serialized_len(&self) -> usize {
+                        std::mem::size_of::<$type>()
+                    }
+
+                    fn write(&self, mut buf: &mut [u8]) {
+                        buf.[< put_ $type>](*self)
+                    }
+
+                    fn read(mut buf: &[u8]) -> Self {
+                        buf.[< get_ $type>]()
+                    }
+                }
+            )*
+        }
+    };
+}
+
+for_all_primitives! { impl_key }
+
+impl Value for Vec<u8> {
+    fn serialized_len(&self) -> usize {
+        self.len()
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        buf.put_slice(self);
+    }
+
+    fn read(buf: &[u8]) -> Self {
+        buf.to_vec()
+    }
+}
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index bfd1c5dc..2877c496 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -14,88 +14,6 @@
 
 #![feature(trait_alias)]
 
-use bytes::{Buf, BufMut};
-
-pub trait Key:
-    Sized
-    + Send
-    + Sync
-    + 'static
-    + std::hash::Hash
-    + Eq
-    + PartialEq
-    + Ord
-    + PartialOrd
-    + Clone
-    + std::fmt::Debug
-{
-    fn serialized_len(&self) -> usize {
-        panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
-    }
-
-    fn write(&self, _buf: &mut [u8]) {
-        panic!("Method `write` must be implemented for `Key` if storage is used.")
-    }
-
-    fn read(_buf: &[u8]) -> Self {
-        panic!("Method `read` must be implemented for `Key` if storage is used.")
-    }
-}
-
-pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
-    fn serialized_len(&self) -> usize {
-        panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
-    }
-
-    fn write(&self, _buf: &mut [u8]) {
-        panic!("Method `write` must be implemented for `Value` if storage is used.")
-    }
-
-    fn read(_buf: &[u8]) -> Self {
-        panic!("Method `read` must be implemented for `Value` if storage is used.")
-    }
-}
-
-// TODO(MrCroxx): use macro to impl for all
-
-impl Key for u64 {
-    fn serialized_len(&self) -> usize {
-        8
-    }
-
-    fn write(&self, mut buf: &mut [u8]) {
-        buf.put_u64(*self);
-    }
-
-    fn read(mut buf: &[u8]) -> Self {
-        buf.get_u64()
-    }
-}
-
-impl Key for u32 {
-    fn serialized_len(&self) -> usize {
-        4
-    }
-
-    fn write(&self, mut buf: &mut [u8]) {
-        buf.put_u32(*self);
-    }
-
-    fn read(mut buf: &[u8]) -> Self {
-        buf.get_u32()
-    }
-}
-
-impl Value for Vec<u8> {
-    fn serialized_len(&self) -> usize {
-        self.len()
-    }
-
-    fn write(&self, mut buf: &mut [u8]) {
-        buf.put_slice(self);
-    }
-
-    fn read(buf: &[u8]) -> Self {
-        buf.to_vec()
-    }
-}
+pub mod bits;
+pub mod code;
+pub mod queue;
diff --git a/foyer-utils/src/queue.rs b/foyer-common/src/queue.rs
similarity index 100%
rename from foyer-utils/src/queue.rs
rename to foyer-common/src/queue.rs
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 10ab82f4..cbc24ed2 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -14,7 +14,7 @@
 
 use std::fmt::Debug;
 
-use foyer_common::Key;
+use foyer_common::code::Key;
 
 use crate::core::pointer::PointerOps;
 
diff --git a/foyer-policy/Cargo.toml b/foyer-policy/Cargo.toml
deleted file mode 100644
index b72ca73b..00000000
--- a/foyer-policy/Cargo.toml
+++ /dev/null
@@ -1,32 +0,0 @@
-[package]
-name = "foyer-policy"
-version = "0.1.0"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
-license = "Apache-2.0"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-bytes = "1"
-cmsketch = "0.1"
-foyer-utils = { path = "../foyer-utils" }
-itertools = "0.10.5"
-memoffset = "0.8"
-parking_lot = "0.12"
-paste = "1.0"
-prometheus = "0.13"
-rand = "0.8.5"
-thiserror = "1"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
-tracing = "0.1"
-twox-hash = "1"
-
-
diff --git a/foyer-policy/src/eviction/lru.rs b/foyer-policy/src/eviction/lru.rs
deleted file mode 100644
index 2c3f33f6..00000000
--- a/foyer-policy/src/eviction/lru.rs
+++ /dev/null
@@ -1,338 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{ptr::NonNull, time::SystemTime};
-
-use foyer_utils::{
-    dlist::{DList, Entry, Iter},
-    intrusive_dlist,
-};
-
-use super::Item;
-
-#[derive(Clone, Debug)]
-pub struct Config {
-    /// Insertion point of the new entry, between 0 and 1.
-    pub lru_insertion_point_fraction: f64,
-}
-
-pub struct Handle<T: Item> {
-    entry: Entry,
-
-    is_in_cache: bool,
-    is_accessed: bool,
-    update_time: SystemTime,
-
-    is_in_tail: bool,
-
-    item: T,
-}
-
-impl<T: Item> Handle<T> {
-    fn new(item: T) -> Self {
-        Self {
-            entry: Entry::default(),
-
-            is_in_cache: false,
-            is_accessed: false,
-            update_time: SystemTime::now(),
-
-            is_in_tail: false,
-
-            item,
-        }
-    }
-}
-
-intrusive_dlist! { Handle<T: Item>, entry, HandleDListAdapter}
-
-pub struct Lru<T: Item> {
-    /// lru list
-    lru: DList<Handle<T>, HandleDListAdapter>,
-
-    /// insertion point
-    insertion_point: Option<NonNull<Handle<T>>>,
-
-    /// length of tail after insertion point
-    tail_len: usize,
-
-    config: Config,
-}
-
-impl<T: Item> Lru<T> {
-    fn new(config: Config) -> Self {
-        Self {
-            lru: DList::new(),
-
-            insertion_point: None,
-
-            tail_len: 0,
-
-            config,
-        }
-    }
-
-    /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
-    /// returns `false` otherwise.
-    fn access(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
-        unsafe {
-            handle.as_mut().is_accessed = true;
-
-            // TODO(MrCroxx): try trigger reconfigure
-
-            self.ensuer_not_insertion_point(handle);
-
-            if handle.as_ref().is_in_cache {
-                self.lru.move_to_head(handle);
-                handle.as_mut().update_time = SystemTime::now();
-            }
-
-            if handle.as_ref().is_in_tail {
-                handle.as_mut().is_in_tail = false;
-                self.tail_len -= 1;
-                self.update_lru_insertion_point();
-            }
-
-            true
-        }
-    }
-
-    /// Returns `true` if handle is successfully added into the lru,
-    /// returns `false` if the handle is already in the lru.
-    fn insert(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
-        unsafe {
-            if handle.as_ref().is_in_cache {
-                return false;
-            }
-
-            match self.insertion_point {
-                Some(insertion_point) => self.lru.link_before(insertion_point, handle),
-                None => self.lru.link_at_head(handle),
-            }
-            handle.as_mut().is_in_cache = true;
-            handle.as_mut().update_time = SystemTime::now();
-            handle.as_mut().is_accessed = false;
-            self.update_lru_insertion_point();
-
-            true
-        }
-    }
-
-    /// Returns `true` if handle is successfully removed from the lru,
-    /// returns `false` if the handle is unchanged.
-    fn remove(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
-        unsafe {
-            if !handle.as_ref().is_in_cache {
-                return false;
-            }
-
-            self.ensuer_not_insertion_point(handle);
-            self.lru.remove(handle);
-            handle.as_mut().is_accessed = false;
-            if handle.as_ref().is_in_tail {
-                handle.as_mut().is_in_tail = false;
-                self.tail_len -= 1;
-            }
-
-            true
-        }
-    }
-
-    fn eviction_iter(&self) -> EvictionIter<'_, T> {
-        unsafe {
-            let mut iter = self.lru.iter();
-            iter.tail();
-            EvictionIter { iter }
-        }
-    }
-
-    fn update_lru_insertion_point(&mut self) {
-        unsafe {
-            if self.config.lru_insertion_point_fraction == 0.0 {
-                return;
-            }
-
-            if self.insertion_point.is_none() {
-                self.insertion_point = self.lru.tail();
-                self.tail_len = 0;
-                if let Some(insertion_point) = &mut self.insertion_point {
-                    insertion_point.as_mut().is_in_tail = true;
-                    self.tail_len += 1;
-                }
-            }
-
-            if self.lru.len() <= 1 {
-                return;
-            }
-
-            assert!(self.insertion_point.is_some());
-
-            let expected_tail_len =
-                (self.lru.len() as f64 * (1.0 - self.config.lru_insertion_point_fraction)) as usize;
-
-            let mut curr = self.insertion_point.unwrap();
-            while self.tail_len < expected_tail_len && Some(curr) != self.lru.head() {
-                curr = self.lru.prev(curr).unwrap();
-                curr.as_mut().is_in_tail = true;
-                self.tail_len += 1;
-            }
-            while self.tail_len > expected_tail_len && Some(curr) != self.lru.tail() {
-                curr.as_mut().is_in_tail = false;
-                self.tail_len -= 1;
-                curr = self.lru.next(curr).unwrap();
-            }
-
-            self.insertion_point = Some(curr);
-        }
-    }
-
-    fn ensuer_not_insertion_point(&mut self, handle: NonNull<Handle<T>>) {
-        unsafe {
-            if Some(handle) == self.insertion_point {
-                self.insertion_point = self.lru.prev(handle);
-                match &mut self.insertion_point {
-                    Some(insertion_point) => {
-                        self.tail_len += 1;
-                        insertion_point.as_mut().is_in_tail = true;
-                    }
-                    // TODO(MrCroxx): think ?
-                    None => assert_eq!(self.lru.len(), 1),
-                }
-            }
-        }
-    }
-}
-
-pub struct EvictionIter<'a, T: Item> {
-    iter: Iter<'a, Handle<T>, HandleDListAdapter>,
-}
-
-impl<'a, T: Item> Iterator for EvictionIter<'a, T> {
-    type Item = &'a T;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        unsafe {
-            match self.iter.element() {
-                Some(element) => {
-                    self.iter.prev();
-                    Some(&element.as_ref().item)
-                }
-                None => None,
-            }
-        }
-    }
-}
-
-// unsafe impl `Send + Sync` for structs with `NonNull` usage
-
-unsafe impl<T: Item> Send for Lru<T> {}
-unsafe impl<T: Item> Sync for Lru<T> {}
-
-unsafe impl<T: Item> Send for Handle<T> {}
-unsafe impl<T: Item> Sync for Handle<T> {}
-
-unsafe impl<'a, T: Item> Send for EvictionIter<'a, T> {}
-unsafe impl<'a, T: Item> Sync for EvictionIter<'a, T> {}
-
-impl super::Config for Config {}
-
-impl<T: Item> super::Handle for Handle<T> {
-    type T = T;
-
-    fn new(item: Self::T) -> Self {
-        Self::new(item)
-    }
-
-    fn item(&self) -> &Self::T {
-        &self.item
-    }
-}
-
-impl<T: Item> super::Policy for Lru<T> {
-    type T = T;
-    type C = Config;
-    type H = Handle<T>;
-    type E<'e> = EvictionIter<'e, T>;
-
-    fn new(config: Self::C) -> Self {
-        Lru::new(config)
-    }
-
-    fn insert(&mut self, handle: NonNull<Self::H>) -> bool {
-        self.insert(handle)
-    }
-
-    fn remove(&mut self, handle: NonNull<Self::H>) -> bool {
-        self.remove(handle)
-    }
-
-    fn access(&mut self, handle: NonNull<Self::H>) -> bool {
-        self.access(handle)
-    }
-
-    fn eviction_iter(&self) -> Self::E<'_> {
-        self.eviction_iter()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use itertools::Itertools;
-
-    use super::*;
-
-    fn ptr<T: Item>(handle: &mut Handle<T>) -> NonNull<Handle<T>> {
-        unsafe { NonNull::new_unchecked(handle as *mut _) }
-    }
-
-    #[test]
-    fn test_lru_simple() {
-        let config = Config {
-            lru_insertion_point_fraction: 0.0,
-        };
-        let mut lru: Lru<u64> = Lru::new(config);
-
-        let mut handles = vec![Handle::new(0), Handle::new(1), Handle::new(2)];
-
-        lru.insert(ptr(&mut handles[0]));
-        lru.insert(ptr(&mut handles[1]));
-        lru.insert(ptr(&mut handles[2]));
-
-        assert_eq!(vec![0, 1, 2], lru.eviction_iter().copied().collect_vec());
-
-        lru.access(ptr(&mut handles[1]));
-
-        assert_eq!(vec![0, 2, 1], lru.eviction_iter().copied().collect_vec());
-
-        lru.remove(ptr(&mut handles[2]));
-
-        assert_eq!(vec![0, 1], lru.eviction_iter().copied().collect_vec());
-
-        drop(handles);
-    }
-}
diff --git a/foyer-policy/src/eviction/mod.rs b/foyer-policy/src/eviction/mod.rs
deleted file mode 100644
index cbcf6eac..00000000
--- a/foyer-policy/src/eviction/mod.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//! A collection of cache eviction policies.
-//!
-//! Cache eviction policies only cares about the order of cached entries.
-//! They don't store the real cache entries or resource usage.
-
-pub mod lru;
-pub mod tinylfu;
-
-use std::ptr::NonNull;
-
-use crate::Item;
-
-pub trait Policy: Send + Sync + 'static {
-    type T: Item;
-    type C: Config;
-    type H: Handle<T = Self::T>;
-    type E<'e>: Iterator<Item = &'e Self::T>;
-
-    fn new(config: Self::C) -> Self;
-
-    fn insert(&mut self, handle: NonNull<Self::H>) -> bool;
-
-    fn remove(&mut self, handle: NonNull<Self::H>) -> bool;
-
-    fn access(&mut self, handle: NonNull<Self::H>) -> bool;
-
-    fn eviction_iter(&self) -> Self::E<'_>;
-}
-
-pub trait Config: Send + Sync + std::fmt::Debug + Clone + 'static {}
-
-pub trait Handle: Send + Sync + 'static {
-    type T: Item;
-
-    fn new(index: Self::T) -> Self;
-
-    fn item(&self) -> &Self::T;
-}
diff --git a/foyer-policy/src/eviction/tinylfu.rs b/foyer-policy/src/eviction/tinylfu.rs
deleted file mode 100644
index 5b87e0ea..00000000
--- a/foyer-policy/src/eviction/tinylfu.rs
+++ /dev/null
@@ -1,493 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{hash::Hasher, ptr::NonNull, time::SystemTime};
-
-use cmsketch::CMSketchUsize;
-use twox_hash::XxHash64;
-
-use foyer_utils::{
-    dlist::{DList, Entry, Iter},
-    intrusive_dlist,
-};
-
-use super::Item;
-
-const MIN_CAPACITY: usize = 100;
-const ERROR_THRESHOLD: f64 = 5.0;
-const HASH_COUNT: usize = 4;
-const DECAY_FACTOR: f64 = 0.5;
-
-#[derive(Clone, Debug)]
-pub struct Config {
-    /// The multiplier for window len given the cache size.
-    pub window_to_cache_size_ratio: usize,
-
-    /// The ratio of tiny lru capacity to overall capacity.
-    pub tiny_lru_capacity_ratio: f64,
-}
-
-#[derive(PartialEq, Eq, Debug)]
-enum LruType {
-    Tiny,
-    Main,
-}
-
-pub struct Handle<T: Item> {
-    entry_tiny: Entry,
-    entry_main: Entry,
-
-    is_in_cache: bool,
-    is_accessed: bool,
-    update_time: SystemTime,
-
-    lru_type: LruType,
-
-    item: T,
-}
-
-impl<T: Item> Handle<T> {
-    fn new(item: T) -> Self {
-        Self {
-            entry_tiny: Entry::default(),
-            entry_main: Entry::default(),
-
-            is_in_cache: false,
-            is_accessed: false,
-            update_time: SystemTime::now(),
-
-            lru_type: LruType::Tiny,
-
-            item,
-        }
-    }
-}
-
-intrusive_dlist! { Handle<T: Item>, entry_tiny, HandleDListTinyAdapter}
-intrusive_dlist! { Handle<T: Item>, entry_main, HandleDListMainAdapter}
-
-/// Implements the W-TinyLFU cache eviction policy as described in -
-///
-/// https://arxiv.org/pdf/1512.00727.pdf
-///
-/// The cache is split into 2 parts, the main cache and the tiny cache.
-/// The tiny cache is typically sized to be 1% of the total cache with
-/// the main cache being the rest 99%. Both caches are implemented using
-/// LRUs. New items land in tiny cache. During eviction, the tail item
-/// from the tiny cache is promoted to main cache if its frequency is
-/// higher than the tail item of of main cache, and the tail of main
-/// cache is evicted. This gives the frequency based admission into main
-/// cache. Hits in each cache simply move the item to the head of each
-/// LRU cache.
-/// The frequency counts are maintained in count-min-sketch approximate
-/// counters -
-///
-/// Counter Overhead:
-/// The window_to_cache_size_ratio determines the size of counters.
-/// The default value is 32 which means the counting window size is
-/// 32 times the cache size. After every 32 X cache capacity number
-/// of items, the counts are halved to weigh frequency by recency.
-/// The function counter_size() returns the size of the counters
-/// in bytes. See maybe_grow_access_counters() implementation for
-/// how the size is computed.
-///
-/// Tiny cache size:
-/// This default to 1%. There's no need to tune this parameter.
-pub struct TinyLfu<T: Item> {
-    /// tiny lru list
-    lru_tiny: DList<Handle<T>, HandleDListTinyAdapter>,
-
-    /// main lru list
-    lru_main: DList<Handle<T>, HandleDListMainAdapter>,
-
-    /// the window length counter
-    window_size: usize,
-
-    /// maxumum value of window length which when hit the counters are halved
-    max_window_size: usize,
-
-    /// the capacity for which the counters are sized
-    capacity: usize,
-
-    /// approximate streaming frequency counters
-    ///
-    /// the counts are halved every time the max_window_len is hit
-    frequencies: CMSketchUsize,
-
-    config: Config,
-}
-
-impl<T: Item> TinyLfu<T> {
-    pub fn new(config: Config) -> Self {
-        let mut res = Self {
-            lru_tiny: DList::new(),
-            lru_main: DList::new(),
-
-            window_size: 0,
-            max_window_size: 0,
-            capacity: 0,
-
-            // A dummy size, will be updated later.
-            frequencies: CMSketchUsize::new_with_size(1, 1),
-
-            config,
-        };
-        res.maybe_grow_access_counters();
-        res
-    }
-
-    /// Returns `true` if the information is recorded and bumped the handle to the head of the lru,
-    /// returns `false` otherwise.
-    fn access(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
-        unsafe {
-            handle.as_mut().is_accessed = true;
-
-            if !handle.as_ref().is_in_cache {
-                return false;
-            }
-
-            match handle.as_ref().lru_type {
-                LruType::Tiny => self.lru_tiny.move_to_head(handle),
-                LruType::Main => self.lru_main.move_to_head(handle),
-            }
-            handle.as_mut().update_time = SystemTime::now();
-            self.update_frequencies(handle);
-
-            true
-        }
-    }
-
-    /// Returns `true` if handle is successfully added into the lru,
-    /// returns `false` if the handle is already in the lru.
-    fn insert(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
-        unsafe {
-            if handle.as_ref().is_in_cache {
-                return false;
-            }
-
-            self.lru_tiny.link_at_head(handle);
-            handle.as_mut().lru_type = LruType::Tiny;
-
-            // Initialize the frequency count for this handle.
-            self.update_frequencies(handle);
-
-            // If tiny cache is full, unconditionally promote tail to main cache.
-            let expected_tiny_len = (self.config.tiny_lru_capacity_ratio
-                * (self.lru_tiny.len() + self.lru_main.len()) as f64)
-                as usize;
-            if self.lru_tiny.len() > expected_tiny_len {
-                let mut handle_tail = self.lru_tiny.tail().unwrap();
-                self.lru_tiny.remove(handle_tail);
-
-                self.lru_main.link_at_head(handle_tail);
-                handle_tail.as_mut().lru_type = LruType::Main;
-            } else {
-                self.maybe_promote_tail();
-            }
-
-            // If the number of counters are too small for the cache size, double them.
-            self.maybe_grow_access_counters();
-
-            handle.as_mut().is_in_cache = true;
-            handle.as_mut().is_accessed = false;
-            handle.as_mut().update_time = SystemTime::now();
-
-            true
-        }
-    }
-
-    /// Returns `true` if handle is successfully removed from the lru,
-    /// returns `false` if the handle is unchanged.
-    fn remove(&mut self, mut handle: NonNull<Handle<T>>) -> bool {
-        unsafe {
-            if !handle.as_ref().is_in_cache {
-                return false;
-            }
-
-            match handle.as_ref().lru_type {
-                LruType::Main => self.lru_main.remove(handle),
-                LruType::Tiny => self.lru_tiny.remove(handle),
-            }
-
-            handle.as_mut().is_accessed = false;
-            handle.as_mut().is_in_cache = false;
-
-            true
-        }
-    }
-
-    fn eviction_iter<'a>(&'a self) -> EvictionIter<'a, T> {
-        unsafe {
-            let mut iter_main: Iter<'a, _, _> = self.lru_main.iter();
-            iter_main.tail();
-            let mut iter_tiny: Iter<'a, _, _> = self.lru_tiny.iter();
-            iter_tiny.tail();
-            EvictionIter {
-                tinylfu: self,
-                iter_main,
-                iter_tiny,
-            }
-        }
-    }
-
-    fn maybe_grow_access_counters(&mut self) {
-        let capacity = self.lru_tiny.len() + self.lru_main.len();
-
-        // If the new capacity ask is more than double the current size,
-        // recreate the approximate frequency counters.
-        if 2 * self.capacity > capacity {
-            return;
-        }
-
-        self.capacity = std::cmp::max(capacity, MIN_CAPACITY);
-
-        // The window counter that's incremented on every fetch.
-        self.window_size = 0;
-
-        // The frequency counters are halved every `max_window_size` fetches to decay the frequency counts.
-        self.max_window_size = self.capacity * self.config.window_to_cache_size_ratio;
-
-        // Number of frequency counters - roughly equal to the window size divided by error tolerance.
-        let num_counters = (1f64.exp() * self.max_window_size as f64 / ERROR_THRESHOLD) as usize;
-        let num_counters = num_counters.next_power_of_two();
-
-        self.frequencies = CMSketchUsize::new_with_size(num_counters, HASH_COUNT);
-    }
-
-    fn update_frequencies(&mut self, handle: NonNull<Handle<T>>) {
-        self.frequencies.record(Self::hash_handle(handle));
-        self.window_size += 1;
-
-        // Decay counts every `max_window_size`. This avoids having items that were
-        // accessed frequently (were hot) but aren't being accessed anymore (are cold)
-        // from staying in cache forever.
-        if self.window_size == self.max_window_size {
-            self.window_size >>= 1;
-            self.frequencies.decay(DECAY_FACTOR);
-        }
-    }
-
-    fn maybe_promote_tail(&mut self) {
-        unsafe {
-            let mut handle_main = match self.lru_main.tail() {
-                Some(handle) => handle,
-                None => return,
-            };
-            let mut handle_tiny = match self.lru_tiny.tail() {
-                Some(handle) => handle,
-                None => return,
-            };
-
-            if self.admit_to_main(handle_main, handle_tiny) {
-                self.lru_tiny.remove(handle_tiny);
-                self.lru_main.link_at_head(handle_tiny);
-                handle_tiny.as_mut().lru_type = LruType::Main;
-
-                self.lru_main.remove(handle_main);
-                self.lru_tiny.link_at_tail(handle_main);
-                handle_main.as_mut().lru_type = LruType::Tiny;
-                return;
-            }
-
-            // A node with high frequency at the tail of main cache might prevent
-            // promotions from tiny cache from happening for a long time. Relocate
-            // the tail of main cache to prevent this.
-            self.lru_main.move_to_head(handle_main);
-        }
-    }
-
-    fn admit_to_main(
-        &self,
-        handle_main: NonNull<Handle<T>>,
-        handle_tiny: NonNull<Handle<T>>,
-    ) -> bool {
-        unsafe {
-            assert_eq!(handle_main.as_ref().lru_type, LruType::Main);
-            assert_eq!(handle_tiny.as_ref().lru_type, LruType::Tiny);
-
-            let frequent_main = self.frequencies.count(Self::hash_handle(handle_main));
-            let frequent_tiny = self.frequencies.count(Self::hash_handle(handle_tiny));
-
-            frequent_main <= frequent_tiny
-        }
-    }
-
-    fn hash_handle(handle: NonNull<Handle<T>>) -> u64 {
-        let mut hasher = XxHash64::default();
-        unsafe { handle.as_ref().item.hash(&mut hasher) };
-        hasher.finish()
-    }
-}
-
-pub struct EvictionIter<'a, T: Item> {
-    tinylfu: &'a TinyLfu<T>,
-    iter_main: Iter<'a, Handle<T>, HandleDListMainAdapter>,
-    iter_tiny: Iter<'a, Handle<T>, HandleDListTinyAdapter>,
-}
-
-impl<'a, T: Item> Iterator for EvictionIter<'a, T> {
-    type Item = &'a T;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        unsafe {
-            let handle_main = self.iter_main.element();
-            let handle_tiny = self.iter_tiny.element();
-
-            match (handle_main, handle_tiny) {
-                (None, None) => None,
-                (Some(handle_main), None) => {
-                    self.iter_main.prev();
-                    Some(&handle_main.as_ref().item)
-                }
-                (None, Some(handle_tiny)) => {
-                    self.iter_tiny.prev();
-                    Some(&handle_tiny.as_ref().item)
-                }
-                (Some(handle_main), Some(handle_tiny)) => {
-                    // Eviction from tiny or main depending on whether the tiny handle woould be
-                    // admitted to main cachce. If it would be, evict from main cache, otherwise
-                    // from tiny cache.
-                    if self.tinylfu.admit_to_main(handle_main, handle_tiny) {
-                        self.iter_main.prev();
-                        Some(&handle_main.as_ref().item)
-                    } else {
-                        self.iter_tiny.prev();
-                        Some(&handle_tiny.as_ref().item)
-                    }
-                }
-            }
-        }
-    }
-}
-
-// unsafe impl `Send + Sync` for structs with `NonNull` usage
-
-unsafe impl<T: Item> Send for TinyLfu<T> {}
-unsafe impl<T: Item> Sync for TinyLfu<T> {}
-
-unsafe impl<T: Item> Send for Handle<T> {}
-unsafe impl<T: Item> Sync for Handle<T> {}
-
-unsafe impl<'a, T: Item> Send for EvictionIter<'a, T> {}
-unsafe impl<'a, T: Item> Sync for EvictionIter<'a, T> {}
-
-impl super::Config for Config {}
-
-impl<T: Item> super::Handle for Handle<T> {
-    type T = T;
-
-    fn new(index: Self::T) -> Self {
-        Self::new(index)
-    }
-
-    fn item(&self) -> &Self::T {
-        &self.item
-    }
-}
-
-impl<T: Item> super::Policy for TinyLfu<T> {
-    type T = T;
-    type C = Config;
-    type H = Handle<T>;
-    type E<'e> = EvictionIter<'e, T>;
-
-    fn new(config: Self::C) -> Self {
-        TinyLfu::new(config)
-    }
-
-    fn insert(&mut self, handle: NonNull<Self::H>) -> bool {
-        self.insert(handle)
-    }
-
-    fn remove(&mut self, handle: NonNull<Self::H>) -> bool {
-        self.remove(handle)
-    }
-
-    fn access(&mut self, handle: NonNull<Self::H>) -> bool {
-        self.access(handle)
-    }
-
-    fn eviction_iter(&self) -> Self::E<'_> {
-        self.eviction_iter()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use itertools::Itertools;
-
-    use super::*;
-
-    fn ptr<T: Item>(handle: &mut Handle<T>) -> NonNull<Handle<T>> {
-        unsafe { NonNull::new_unchecked(handle as *mut _) }
-    }
-
-    #[test]
-    fn test_lru_simple() {
-        let config = Config {
-            window_to_cache_size_ratio: 10,
-            tiny_lru_capacity_ratio: 0.01,
-        };
-        let mut lfu: TinyLfu<u64> = TinyLfu::new(config);
-
-        let mut handles = (0..101).map(Handle::new).collect_vec();
-        for handle in handles.iter_mut().take(100) {
-            lfu.insert(ptr(handle));
-        }
-
-        assert_eq!(99, lfu.lru_main.len());
-        assert_eq!(1, lfu.lru_tiny.len());
-        assert_eq!(handles[0].lru_type, LruType::Tiny);
-
-        // 0 will be evicted at last because it is on tiny lru but its frequency equals to others
-        assert_eq!(
-            (1..100).chain([0].into_iter()).collect_vec(),
-            lfu.eviction_iter().copied().collect_vec()
-        );
-
-        for handle in handles.iter_mut().take(100) {
-            lfu.access(ptr(handle));
-        }
-        lfu.access(ptr(&mut handles[0]));
-        lfu.insert(ptr(&mut handles[100]));
-
-        assert_eq!(handles[0].lru_type, LruType::Main);
-        assert_eq!(handles[100].lru_type, LruType::Tiny);
-
-        assert_eq!(
-            [100]
-                .into_iter()
-                .chain(1..100)
-                .chain([0].into_iter())
-                .collect_vec(),
-            lfu.eviction_iter().copied().collect_vec()
-        );
-
-        drop(handles);
-    }
-}
diff --git a/foyer-policy/src/lib.rs b/foyer-policy/src/lib.rs
deleted file mode 100644
index 4dddce56..00000000
--- a/foyer-policy/src/lib.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#![feature(trait_alias)]
-#![feature(pattern)]
-
-pub mod eviction;
-pub mod reinsertion;
-
-pub trait Item = PartialOrd
-    + Ord
-    + PartialEq
-    + Eq
-    + Clone
-    + std::hash::Hash
-    + Send
-    + Sync
-    + 'static
-    + std::fmt::Debug;
diff --git a/foyer-policy/src/reinsertion/mod.rs b/foyer-policy/src/reinsertion/mod.rs
deleted file mode 100644
index 9717a1ff..00000000
--- a/foyer-policy/src/reinsertion/mod.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-pub trait Policy {}
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index a6be9300..b6a77967 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -14,7 +14,6 @@ bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
-foyer-utils = { path = "../foyer-utils" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
diff --git a/foyer-storage/src/admission.rs b/foyer-storage/src/admission.rs
index ea500f2c..f80cd3d3 100644
--- a/foyer-storage/src/admission.rs
+++ b/foyer-storage/src/admission.rs
@@ -14,7 +14,7 @@
 
 use std::marker::PhantomData;
 
-use foyer_common::{Key, Value};
+use foyer_common::code::{Key, Value};
 
 use std::fmt::Debug;
 
diff --git a/foyer-storage/src/device/io_buffer.rs b/foyer-storage/src/device/io_buffer.rs
index 9046caec..1a974b96 100644
--- a/foyer-storage/src/device/io_buffer.rs
+++ b/foyer-storage/src/device/io_buffer.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use foyer_utils::bits;
+use foyer_common::bits;
 use std::alloc::{Allocator, Global};
 
 #[derive(Debug, Clone, Copy)]
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 33d791cd..76cc6b70 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -14,8 +14,8 @@
 
 use std::sync::Arc;
 
+use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use foyer_utils::queue::AsyncQueue;
 use itertools::Itertools;
 use tokio::sync::{
     mpsc::{channel, error::TrySendError, Receiver, Sender},
diff --git a/foyer-storage/src/indices.rs b/foyer-storage/src/indices.rs
index 9c87641c..37f8223c 100644
--- a/foyer-storage/src/indices.rs
+++ b/foyer-storage/src/indices.rs
@@ -14,7 +14,7 @@
 
 use std::collections::BTreeMap;
 
-use foyer_common::Key;
+use foyer_common::code::Key;
 use itertools::Itertools;
 use parking_lot::RwLock;
 
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 876155c1..883de041 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -24,9 +24,11 @@ use crate::{
     reinsertion::ReinsertionPolicy,
     store::Store,
 };
-use foyer_common::{Key, Value};
+use foyer_common::{
+    code::{Key, Value},
+    queue::AsyncQueue,
+};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use foyer_utils::queue::AsyncQueue;
 use itertools::Itertools;
 use tokio::sync::{
     mpsc::{channel, error::TrySendError, Receiver, Sender},
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 99eca09d..b0770269 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -14,12 +14,12 @@
 
 use std::{marker::PhantomData, sync::Arc};
 
+use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{
     core::{adapter::Link, pointer::PointerOps},
     eviction::EvictionPolicy,
     intrusive_adapter, key_adapter,
 };
-use foyer_utils::queue::AsyncQueue;
 use tokio::sync::RwLock;
 
 use crate::{
diff --git a/foyer-storage/src/reinsertion.rs b/foyer-storage/src/reinsertion.rs
index 49b932b8..96ca5b13 100644
--- a/foyer-storage/src/reinsertion.rs
+++ b/foyer-storage/src/reinsertion.rs
@@ -14,7 +14,7 @@
 
 use std::marker::PhantomData;
 
-use foyer_common::{Key, Value};
+use foyer_common::code::{Key, Value};
 
 use std::fmt::Debug;
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index d0c56872..cde2ac3e 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -15,8 +15,8 @@
 use std::{fmt::Debug, marker::PhantomData, sync::Arc};
 
 use bytes::{Buf, BufMut};
+use foyer_common::{bits::align_up, queue::AsyncQueue};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use foyer_utils::{bits::align_up, queue::AsyncQueue};
 use twox_hash::XxHash64;
 
 use crate::{
@@ -30,7 +30,7 @@ use crate::{
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
 };
-use foyer_common::{Key, Value};
+use foyer_common::code::{Key, Value};
 use std::hash::Hasher;
 
 pub struct StoreConfig<D, AP, EP, RP, EL>
diff --git a/foyer-utils/Cargo.toml b/foyer-utils/Cargo.toml
deleted file mode 100644
index f457935d..00000000
--- a/foyer-utils/Cargo.toml
+++ /dev/null
@@ -1,25 +0,0 @@
-[package]
-name = "foyer-utils"
-version = "0.1.0"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
-license = "Apache-2.0"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-bytes = "1"
-itertools = "0.10.5"
-memoffset = "0.8"
-parking_lot = "0.12"
-paste = "1.0"
-thiserror = "1"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
-
diff --git a/foyer-utils/src/dlist.rs b/foyer-utils/src/dlist.rs
deleted file mode 100644
index 7143c27f..00000000
--- a/foyer-utils/src/dlist.rs
+++ /dev/null
@@ -1,606 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{marker::PhantomData, ptr::NonNull};
-
-/// An entry in an intrusive double linked list
-#[derive(Default, Debug)]
-pub struct Entry {
-    /// The prev entry ptr in the double linked list.
-    prev: Option<NonNull<Entry>>,
-
-    /// The next entry ptr in the double linked list.
-    next: Option<NonNull<Entry>>,
-}
-
-impl Entry {
-    pub fn is_linked(&self) -> bool {
-        self.prev.is_some() || self.next.is_some()
-    }
-}
-
-pub trait Adapter<E> {
-    /// entry ptr to element ptr
-    fn en2el(_: NonNull<Entry>) -> NonNull<E>;
-
-    /// element ptr to entry ptr
-    fn el2en(_: NonNull<E>) -> NonNull<Entry>;
-}
-
-#[macro_export]
-macro_rules! intrusive_dlist {
-    (
-        $element:ident $(< $( $lt:tt $( : $clt:tt $(+ $dlt:tt)* )? ),+ >)?, $entry:ident, $adapter:ident
-    ) => {
-        struct $adapter;
-
-        impl $(< $( $lt $( : $clt $(+ $dlt )* )? ),+ >)? $crate::dlist::Adapter<$element $(< $( $lt ),+ >)?> for $adapter  {
-            fn en2el(entry: std::ptr::NonNull<Entry>) -> std::ptr::NonNull<$element $(< $( $lt ),+ >)?> {
-                unsafe {
-                    let ptr = (entry.as_ptr() as *mut u8)
-                        .sub(memoffset::offset_of!($element $(< $( $lt ),+ >)?, $entry))
-                        .cast::<$element $(< $( $lt ),+ >)?>();
-                    std::ptr::NonNull::new_unchecked(ptr)
-                }
-            }
-
-            fn el2en(element: std::ptr::NonNull<$element $(< $( $lt ),+ >)?>) -> std::ptr::NonNull<Entry> {
-                unsafe {
-                    let ptr = (element.as_ptr() as *mut u8)
-                        .add(memoffset::offset_of!($element $(< $( $lt ),+ >)?, $entry))
-                        .cast::<Entry>();
-                    std::ptr::NonNull::new_unchecked(ptr)
-                }
-            }
-        }
-    };
-}
-
-pub use crate::intrusive_dlist;
-
-/// TODO: write docs
-#[derive(Debug)]
-pub struct DList<E, A: Adapter<E>> {
-    /// head ptr of the double linked list
-    head: Option<NonNull<Entry>>,
-
-    /// tail ptr of the double linked list
-    tail: Option<NonNull<Entry>>,
-
-    /// length of the double linked list
-    len: usize,
-
-    _marker: PhantomData<(E, A)>,
-}
-
-impl<E, A: Adapter<E>> Default for DList<E, A> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<E, A: Adapter<E>> DList<E, A> {
-    /// create an empty intrusive double linklist
-    pub fn new() -> Self {
-        Self {
-            head: None,
-            tail: None,
-            len: 0,
-            _marker: PhantomData::default(),
-        }
-    }
-
-    /// link a dangling element to the list at head
-    ///
-    /// # Safety
-    ///
-    /// `element` must not be in the list
-    pub unsafe fn link_at_head(&mut self, element: NonNull<E>) {
-        let mut entry = A::el2en(element);
-        assert_ne!(Some(entry), self.head);
-
-        entry.as_mut().next = self.head;
-        entry.as_mut().prev = None;
-
-        // fix the prev ptr of head
-        if let Some(head) = &mut self.head {
-            head.as_mut().prev = Some(entry);
-        }
-        self.head = Some(entry);
-
-        if self.tail.is_none() {
-            self.tail = Some(entry)
-        }
-
-        self.len += 1;
-    }
-
-    /// link a dangling element to the list at tail
-    ///
-    /// # Safety
-    ///
-    /// `element` must not be in the list
-    pub unsafe fn link_at_tail(&mut self, element: NonNull<E>) {
-        let mut entry = A::el2en(element);
-        assert_ne!(Some(entry), self.tail);
-
-        entry.as_mut().next = None;
-        entry.as_mut().prev = self.tail;
-
-        // fix the prev ptr of tail
-        if let Some(tail) = &mut self.tail {
-            tail.as_mut().next = Some(entry);
-        }
-        self.tail = Some(entry);
-
-        if self.head.is_none() {
-            self.head = Some(entry)
-        }
-
-        self.len += 1;
-    }
-
-    /// link a dangling element to the list before `next_element`
-    ///
-    /// # Safety
-    ///
-    /// `element` must not be in the list
-    /// `next_element` must be in the list
-    pub unsafe fn link_before(&mut self, next_element: NonNull<E>, element: NonNull<E>) {
-        let mut next_entry = A::el2en(next_element);
-        let mut entry = A::el2en(element);
-        assert_ne!(next_entry, entry);
-
-        let mut prev = next_entry.as_mut().prev;
-        assert_ne!(prev, Some(entry));
-
-        entry.as_mut().prev = prev;
-        match &mut prev {
-            Some(prev) => prev.as_mut().next = Some(entry),
-            None => self.head = Some(entry),
-        }
-
-        next_entry.as_mut().prev = Some(entry);
-        entry.as_mut().next = Some(next_entry);
-
-        self.len += 1;
-    }
-
-    /// unlink an element from the list
-    ///
-    /// # Safety
-    ///
-    /// `element` must be in the list
-    pub unsafe fn unlink(&mut self, element: NonNull<E>) {
-        assert!(self.len > 0);
-
-        let mut entry = A::el2en(element);
-
-        // fix head and tail if node is either of that
-        let mut prev = entry.as_mut().prev;
-        let mut next = entry.as_mut().next;
-        if Some(entry) == self.head {
-            self.head = next;
-        }
-        if Some(entry) == self.tail {
-            self.tail = prev;
-        }
-
-        // fix the next and prev ptrs of the node before and after this
-        if let Some(prev) = &mut prev {
-            prev.as_mut().next = next;
-        }
-        if let Some(next) = &mut next {
-            next.as_mut().prev = prev;
-        }
-
-        self.len -= 1;
-    }
-
-    /// remove an element from the list and cleanup its pointers
-    ///
-    /// # Safety
-    ///
-    /// `element` must be in the list
-    pub unsafe fn remove(&mut self, element: NonNull<E>) {
-        self.unlink(element);
-        let mut entry = A::el2en(element);
-        entry.as_mut().next = None;
-        entry.as_mut().prev = None;
-    }
-
-    /// replace an element with an dangling element
-    ///
-    /// # Safety
-    ///
-    /// `old_element` must be in the list
-    /// `new_element` must not be in the list
-    pub unsafe fn replace(&mut self, old_element: NonNull<E>, new_element: NonNull<E>) {
-        let mut old_entry = A::el2en(old_element);
-        let mut new_entry = A::el2en(new_element);
-
-        // update head and tail links if needed
-        if Some(old_entry) == self.head {
-            self.head = Some(new_entry);
-        }
-        if Some(old_entry) == self.tail {
-            self.tail = Some(new_entry);
-        }
-
-        let mut prev = old_entry.as_mut().prev;
-        let mut next = old_entry.as_mut().next;
-
-        // make the prev and next entry point to the new entry
-        if let Some(prev) = &mut prev {
-            prev.as_mut().next = Some(new_entry);
-        }
-        if let Some(next) = &mut next {
-            next.as_mut().prev = Some(new_entry);
-        }
-
-        // make the new entry point to the prev and next entry
-        new_entry.as_mut().prev = prev;
-        new_entry.as_mut().next = next;
-
-        // cleanup the old node
-        old_entry.as_mut().prev = None;
-        old_entry.as_mut().next = None;
-    }
-
-    /// move an element to the head of the list
-    ///
-    /// # Safety
-    ///
-    /// `element` must be in the list
-    pub unsafe fn move_to_head(&mut self, element: NonNull<E>) {
-        let entry = A::el2en(element);
-        if Some(entry) == self.head {
-            return;
-        }
-        self.unlink(element);
-        self.link_at_head(element);
-    }
-
-    /// move an element to the tail of the list
-    ///
-    /// # Safety
-    ///
-    /// `element` must be in the list
-    pub unsafe fn move_to_tail(&mut self, element: NonNull<E>) {
-        let entry = A::el2en(element);
-        if Some(entry) == self.tail {
-            return;
-        }
-        self.unlink(element);
-        self.link_at_tail(element);
-    }
-
-    pub fn head(&self) -> Option<NonNull<E>> {
-        self.head.map(|entry| A::en2el(entry))
-    }
-
-    pub fn tail(&self) -> Option<NonNull<E>> {
-        self.tail.map(|entry| A::en2el(entry))
-    }
-
-    /// # Safety
-    ///
-    /// element` must be in the list
-    pub unsafe fn prev(&self, element: NonNull<E>) -> Option<NonNull<E>> {
-        let entry = A::el2en(element);
-        entry.as_ref().prev.map(|entry| A::en2el(entry))
-    }
-
-    /// # Safety
-    ///
-    /// element` must be in the list
-    pub unsafe fn next(&self, element: NonNull<E>) -> Option<NonNull<E>> {
-        let entry = A::el2en(element);
-        entry.as_ref().next.map(|entry| A::en2el(entry))
-    }
-
-    pub fn len(&self) -> usize {
-        self.len
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len == 0
-    }
-
-    /// create a iterator of the list from head
-    ///
-    /// # Safety
-    ///
-    /// there is no guarantee that the element cannot be modified while iterating
-    pub unsafe fn iter(&self) -> Iter<'_, E, A> {
-        let entry = self.head;
-        Iter { dlist: self, entry }
-    }
-}
-
-pub struct Iter<'a, E, A: Adapter<E>> {
-    dlist: &'a DList<E, A>,
-    entry: Option<NonNull<Entry>>,
-}
-
-impl<'a, E, A: Adapter<E>> Iter<'a, E, A> {
-    /// get element ptr
-    pub fn element(&self) -> Option<NonNull<E>> {
-        self.entry.map(|entry| A::en2el(entry))
-    }
-
-    /// move to next
-    ///
-    /// # Safety
-    ///
-    /// the iterator must be in a valid position
-    pub unsafe fn next(&mut self) {
-        self.entry = self.entry.unwrap().as_ref().next
-    }
-
-    /// move to prev
-    ///
-    /// # Safety
-    ///
-    /// the iterator must be in a valid position
-    pub unsafe fn prev(&mut self) {
-        self.entry = self.entry.unwrap().as_ref().prev
-    }
-
-    /// move to head
-    pub fn head(&mut self) {
-        self.entry = self.dlist.head
-    }
-
-    /// move to tail
-    pub fn tail(&mut self) {
-        self.entry = self.dlist.tail
-    }
-}
-
-impl<'a, E, A: Adapter<E>> Iterator for Iter<'a, E, A> {
-    type Item = NonNull<E>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.element() {
-            Some(element) => {
-                unsafe { self.next() };
-                Some(element)
-            }
-            None => None,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use itertools::Itertools;
-
-    #[derive(Debug)]
-    struct SingleElement {
-        e: Entry,
-        data: usize,
-    }
-
-    impl SingleElement {
-        fn new(data: usize) -> Self {
-            Self {
-                e: Entry::default(),
-                data,
-            }
-        }
-    }
-
-    impl Drop for SingleElement {
-        fn drop(&mut self) {
-            self.data = 0
-        }
-    }
-
-    #[derive(Debug)]
-    struct DoubleElement {
-        e1: Entry,
-        e2: Entry,
-        data: usize,
-    }
-
-    impl DoubleElement {
-        fn new(data: usize) -> Self {
-            Self {
-                e1: Entry::default(),
-                e2: Entry::default(),
-                data,
-            }
-        }
-    }
-
-    impl Drop for DoubleElement {
-        fn drop(&mut self) {
-            self.data = 0
-        }
-    }
-
-    intrusive_dlist! {SingleElement, e, SingleElementAdapter}
-    intrusive_dlist! {DoubleElement, e1, DoubleElementAdapter1}
-    intrusive_dlist! {DoubleElement, e2, DEA2DoubleElementAdapter2}
-
-    #[test]
-    fn test_se_simple() {
-        unsafe {
-            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
-
-            let mut es = vec![
-                SingleElement::new(1),
-                SingleElement::new(2),
-                SingleElement::new(3),
-            ];
-
-            es.iter_mut()
-                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
-
-            assert_eq!(l.len(), 3);
-            assert_eq!(
-                vec![1, 2, 3],
-                l.iter().map(|e| e.as_ref().data).collect_vec()
-            );
-
-            drop(es);
-        }
-    }
-
-    #[test]
-    fn test_de_simple() {
-        unsafe {
-            let mut l1: DList<DoubleElement, DoubleElementAdapter1> = DList::new();
-            let mut l2: DList<DoubleElement, DEA2DoubleElementAdapter2> = DList::new();
-
-            let mut es = vec![
-                DoubleElement::new(1),
-                DoubleElement::new(2),
-                DoubleElement::new(3),
-            ];
-            es.iter_mut()
-                .for_each(|e| l1.link_at_tail(NonNull::new_unchecked(e as *mut _)));
-            es.iter_mut()
-                .for_each(|e| l2.link_at_head(NonNull::new_unchecked(e as *mut _)));
-
-            assert_eq!(l1.len(), 3);
-            assert_eq!(l2.len(), 3);
-
-            assert_eq!(
-                vec![1, 2, 3],
-                l1.iter().map(|e| e.as_ref().data).collect_vec()
-            );
-            assert_eq!(
-                vec![3, 2, 1],
-                l2.iter().map(|e| e.as_ref().data).collect_vec()
-            );
-
-            drop(es);
-        }
-    }
-
-    #[test]
-    fn test_link_before() {
-        unsafe {
-            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
-
-            let mut es = vec![
-                SingleElement::new(1),
-                SingleElement::new(2),
-                SingleElement::new(3),
-            ];
-
-            es.iter_mut()
-                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
-
-            let mut e = SingleElement::new(4);
-            l.link_before(
-                NonNull::new_unchecked(&mut es[2] as *mut _),
-                NonNull::new_unchecked(&mut e as *mut _),
-            );
-
-            assert_eq!(l.len(), 4);
-            assert_eq!(
-                vec![1, 2, 4, 3],
-                l.iter().map(|e| e.as_ref().data).collect_vec()
-            );
-
-            drop(e);
-            drop(es);
-        }
-    }
-
-    #[test]
-    fn test_remove() {
-        unsafe {
-            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
-
-            let mut es = vec![
-                SingleElement::new(1),
-                SingleElement::new(2),
-                SingleElement::new(3),
-            ];
-
-            es.iter_mut()
-                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
-
-            l.remove(NonNull::new_unchecked(&mut es[1] as *mut _));
-
-            assert_eq!(l.len(), 2);
-            assert_eq!(vec![1, 3], l.iter().map(|e| e.as_ref().data).collect_vec());
-
-            drop(es);
-        }
-    }
-
-    #[test]
-    fn test_link_replace() {
-        unsafe {
-            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
-
-            let mut es = vec![
-                SingleElement::new(1),
-                SingleElement::new(2),
-                SingleElement::new(3),
-            ];
-
-            es.iter_mut()
-                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
-
-            let mut e = SingleElement::new(4);
-            l.replace(
-                NonNull::new_unchecked(&mut es[1] as *mut _),
-                NonNull::new_unchecked(&mut e as *mut _),
-            );
-
-            assert_eq!(l.len(), 3);
-            assert_eq!(
-                vec![1, 4, 3],
-                l.iter().map(|e| e.as_ref().data).collect_vec()
-            );
-
-            drop(e);
-            drop(es);
-        }
-    }
-
-    #[test]
-    fn test_move_to_head() {
-        unsafe {
-            let mut l: DList<SingleElement, SingleElementAdapter> = DList::new();
-
-            let mut es = vec![
-                SingleElement::new(1),
-                SingleElement::new(2),
-                SingleElement::new(3),
-            ];
-
-            es.iter_mut()
-                .for_each(|e| l.link_at_tail(NonNull::new_unchecked(e as *mut _)));
-
-            l.move_to_head(NonNull::new_unchecked(&mut es[1] as *mut _));
-
-            assert_eq!(l.len(), 3);
-            assert_eq!(
-                vec![2, 1, 3],
-                l.iter().map(|e| e.as_ref().data).collect_vec()
-            );
-
-            drop(es);
-        }
-    }
-}
diff --git a/foyer-utils/src/lib.rs b/foyer-utils/src/lib.rs
deleted file mode 100644
index 6c953bb6..00000000
--- a/foyer-utils/src/lib.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#![feature(trait_alias)]
-
-pub mod bits;
-pub mod dlist;
-pub mod queue;
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index e33dfddc..c9bdf5b4 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -13,8 +13,6 @@ bytes = "1"
 cmsketch = "0.1"
 crossbeam = "0.8"
 foyer-common = { path = "../foyer-common" }
-foyer-policy = { path = "../foyer-policy" }
-foyer-utils = { path = "../foyer-utils" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
diff --git a/foyer/src/container.rs b/foyer/src/container.rs
deleted file mode 100644
index 888654fa..00000000
--- a/foyer/src/container.rs
+++ /dev/null
@@ -1,355 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{collections::BTreeMap, hash::Hasher};
-
-use std::{ptr::NonNull, sync::Arc};
-
-use futures::future::try_join_all;
-use itertools::Itertools;
-use tokio::sync::{Mutex, MutexGuard};
-use twox_hash::XxHash64;
-
-use crate::{store::Store, Data, Index, Metrics, WrappedNonNull};
-use foyer_policy::eviction::{Handle, Policy};
-
-// TODO(MrCroxx): wrap own result type
-use crate::store::error::Result;
-
-pub struct Config<I, P, H, S>
-where
-    I: Index,
-    P: Policy<T = I, H = H>,
-    H: Handle<T = I>,
-    S: Store<I = I>,
-{
-    pub capacity: usize,
-
-    pub pool_count_bits: usize,
-
-    pub policy_config: P::C,
-
-    pub store_config: S::C,
-}
-
-impl<I, P, H, S> std::fmt::Debug for Config<I, P, H, S>
-where
-    I: Index,
-    P: Policy<T = I, H = H>,
-    H: Handle<T = I>,
-    S: Store<I = I>,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Config")
-            .field("capacity", &self.capacity)
-            .field("pool_count_bits", &self.pool_count_bits)
-            .field("policy_config", &self.policy_config)
-            .field("store_config", &self.store_config)
-            .finish()
-    }
-}
-
-#[allow(clippy::type_complexity)]
-pub struct Container<I, P, H, D, S>
-where
-    I: Index,
-    P: Policy<T = I, H = H>,
-    H: Handle<T = I>,
-    D: Data,
-    S: Store<I = I, D = D>,
-{
-    pool_count_bits: usize,
-    pools: Vec<Mutex<Pool<I, P, H, D, S>>>,
-
-    metrics: Arc<Metrics>,
-}
-
-impl<I, P, H, D, S> Container<I, P, H, D, S>
-where
-    I: Index,
-    P: Policy<T = I, H = H>,
-    H: Handle<T = I>,
-    D: Data,
-
-    S: Store<I = I, D = D>,
-{
-    pub async fn open(config: Config<I, P, H, S>) -> Result<Self> {
-        Self::open_with_registry(config, prometheus::Registry::new()).await
-    }
-
-    pub async fn open_with_registry(
-        config: Config<I, P, H, S>,
-        registry: prometheus::Registry,
-    ) -> Result<Self> {
-        tracing::info!("open foyer with config: \n{:#?}", config);
-
-        let pool_count = 1 << config.pool_count_bits;
-        let capacity = config.capacity >> config.pool_count_bits;
-
-        let metrics = Arc::new(Metrics::new(registry));
-
-        let stores = (0..pool_count)
-            .map(|pool| S::open(pool, config.store_config.clone(), metrics.clone()))
-            .collect_vec();
-        let stores = try_join_all(stores).await?;
-
-        let pools = stores
-            .into_iter()
-            .enumerate()
-            .map(|(id, store)| Pool {
-                _id: id,
-                policy: P::new(config.policy_config.clone()),
-                capacity,
-                size: 0,
-                handles: BTreeMap::new(),
-                store,
-                _metrics: metrics.clone(),
-            })
-            .map(Mutex::new)
-            .collect_vec();
-
-        Ok(Self {
-            pool_count_bits: config.pool_count_bits,
-            pools,
-            metrics,
-        })
-    }
-
-    pub async fn insert(&self, index: I, data: D) -> Result<bool> {
-        let _timer = self.metrics.latency_insert.start_timer();
-
-        let mut pool = self.pool(&index).await;
-
-        if pool.handles.get(&index).is_some() {
-            // already in cache
-            return Ok(false);
-        }
-
-        let weight = data.weight();
-
-        pool.make_room(weight).await?;
-
-        pool.insert(index, weight, data).await?;
-
-        Ok(true)
-    }
-
-    pub async fn remove(&self, index: &I) -> Result<bool> {
-        let _timer = self.metrics.latency_remove.start_timer();
-
-        let mut pool = self.pool(index).await;
-
-        if pool.handles.get(index).is_none() {
-            // not in cache
-            return Ok(false);
-        }
-
-        pool.remove(index).await?;
-
-        Ok(true)
-    }
-
-    pub async fn get(&self, index: &I) -> Result<Option<D>> {
-        let _timer = self.metrics.latency_get.start_timer();
-
-        let mut pool = self.pool(index).await;
-
-        let res = pool.get(index).await?;
-
-        if res.is_none() {
-            self.metrics.miss.inc();
-        }
-
-        Ok(res)
-    }
-
-    // TODO(MrCroxx): optimize this
-    pub async fn size(&self) -> usize {
-        let mut size = 0;
-        for pool in &self.pools {
-            size += pool.lock().await.size
-        }
-        size
-    }
-
-    async fn pool(&self, index: &I) -> MutexGuard<'_, Pool<I, P, H, D, S>> {
-        let mut hasher = XxHash64::default();
-        index.hash(&mut hasher);
-        let id = hasher.finish() as usize & ((1 << self.pool_count_bits) - 1);
-
-        self.pools[id].lock().await
-    }
-}
-
-struct Pool<I, P, H, D, S>
-where
-    I: Index,
-    P: Policy<T = I, H = H>,
-    H: Handle<T = I>,
-    D: Data,
-    S: Store<I = I, D = D>,
-{
-    _id: usize,
-
-    policy: P,
-
-    capacity: usize,
-
-    size: usize,
-
-    handles: BTreeMap<I, PoolHandle<I, H>>,
-
-    store: S,
-
-    _metrics: Arc<Metrics>,
-}
-
-impl<I, P, H, D, S> Pool<I, P, H, D, S>
-where
-    I: Index,
-    P: Policy<T = I, H = H>,
-    H: Handle<T = I>,
-    D: Data,
-    S: Store<I = I, D = D>,
-{
-    async fn make_room(&mut self, weight: usize) -> Result<()> {
-        let mut handles = vec![];
-        for index in self.policy.eviction_iter() {
-            if self.size + weight <= self.capacity || self.size == 0 {
-                break;
-            }
-            let pool_handle = self.handles.remove(index).unwrap();
-            self.size -= pool_handle.weight;
-            handles.push(pool_handle.handle);
-            self.store.delete(index).await?;
-        }
-        for handle in handles {
-            assert!(self.policy.remove(handle.0));
-            unsafe { drop(Box::from_raw(handle.0.as_ptr())) };
-        }
-        Ok(())
-    }
-
-    async fn insert(&mut self, index: I, weight: usize, data: D) -> Result<()> {
-        let handle = Box::new(H::new(index.clone()));
-        let handle = unsafe { WrappedNonNull(NonNull::new_unchecked(Box::leak(handle))) };
-
-        assert!(self.policy.insert(handle.0));
-        self.handles
-            .insert(index.clone(), PoolHandle { weight, handle });
-        self.size += weight;
-
-        self.store.store(index, data).await?;
-
-        Ok(())
-    }
-
-    async fn remove(&mut self, index: &I) -> Result<()> {
-        let PoolHandle { weight, handle } = self.handles.remove(index).unwrap();
-        assert!(self.policy.remove(handle.0));
-        self.size -= weight;
-
-        self.store.delete(index).await?;
-
-        Ok(())
-    }
-
-    async fn get(&mut self, index: &I) -> Result<Option<D>> {
-        match self.handles.get(index) {
-            Some(pool_handle) => {
-                self.policy.access(pool_handle.handle.0);
-                self.store.load(index).await
-            }
-            None => Ok(None),
-        }
-    }
-}
-
-impl<I, P, H, D, S> Drop for Pool<I, P, H, D, S>
-where
-    I: Index,
-    P: Policy<T = I, H = H>,
-    H: Handle<T = I>,
-    D: Data,
-    S: Store<I = I, D = D>,
-{
-    fn drop(&mut self) {
-        let mut handles = BTreeMap::new();
-        std::mem::swap(&mut handles, &mut self.handles);
-        for PoolHandle { weight: _, handle } in handles.into_values() {
-            unsafe { drop(Box::from_raw(handle.0.as_ptr())) }
-        }
-    }
-}
-
-struct PoolHandle<I, H>
-where
-    I: Index,
-    H: Handle<T = I>,
-{
-    weight: usize,
-    // Use `WrappedNonNull` for ptrs that needs to cross .await points.
-    handle: WrappedNonNull<H>,
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use crate::{store::tests::MemoryStore, tests::is_send_sync_static};
-    use foyer_policy::eviction::{
-        lru::{Config as LruConfig, Handle as LruHandle, Lru},
-        tinylfu::Handle as TinyLfuHandle,
-    };
-
-    #[tokio::test]
-    async fn test_container_simple() {
-        is_send_sync_static::<PoolHandle<u64, LruHandle<u64>>>();
-        is_send_sync_static::<PoolHandle<u64, TinyLfuHandle<u64>>>();
-
-        let policy_config = LruConfig {
-            lru_insertion_point_fraction: 0.0,
-        };
-
-        let config = Config {
-            capacity: 100,
-            pool_count_bits: 0,
-            policy_config,
-
-            store_config: (),
-        };
-
-        let container: Container<u64, Lru<_>, LruHandle<_>, Vec<u8>, MemoryStore<_, _>> =
-            Container::open(config).await.unwrap();
-
-        assert!(container.insert(1, vec![b'x'; 40]).await.unwrap());
-        assert!(!container.insert(1, vec![b'x'; 40]).await.unwrap());
-        assert_eq!(container.get(&1).await.unwrap(), Some(vec![b'x'; 40]));
-
-        assert!(container.insert(2, vec![b'x'; 60]).await.unwrap());
-        assert!(!container.insert(2, vec![b'x'; 60]).await.unwrap());
-        assert_eq!(container.get(&2).await.unwrap(), Some(vec![b'x'; 60]));
-
-        // After insert 3, {1, 2} will be evicted.
-        assert!(container.insert(3, vec![b'x'; 50]).await.unwrap());
-        assert_eq!(container.size().await, 50);
-        assert_eq!(container.get(&3).await.unwrap(), Some(vec![b'x'; 50]));
-        assert_eq!(container.get(&1).await.unwrap(), None);
-        assert_eq!(container.get(&2).await.unwrap(), None);
-
-        assert!(container.remove(&3).await.unwrap());
-
-        assert_eq!(container.size().await, 0);
-    }
-}
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 96ba81c0..213a7b3f 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -15,13 +15,7 @@
 #![feature(trait_alias)]
 #![feature(pattern)]
 
-use std::{fmt::Debug, ptr::NonNull};
-
-use paste::paste;
-
-mod container;
 mod metrics;
-mod store;
 
 pub trait Weight {
     fn weight(&self) -> usize;
@@ -32,98 +26,3 @@ impl Weight for Vec<u8> {
         self.len()
     }
 }
-
-pub trait Index:
-    PartialOrd + Ord + PartialEq + Eq + Clone + std::hash::Hash + Send + Sync + 'static + Debug
-{
-    fn size() -> usize;
-
-    fn write(&self, buf: &mut [u8]);
-
-    fn read(buf: &[u8]) -> Self;
-}
-pub trait Data = Send + Sync + 'static + Into<Vec<u8>> + From<Vec<u8>> + Weight;
-
-macro_rules! for_all_primitives {
-    ($macro:ident) => {
-        $macro! {
-            u8, u16, u32, u64,
-            i8, i16, i32, i64,
-        }
-    };
-}
-
-macro_rules! impl_index {
-    ($( $type:ty, )*) => {
-        use bytes::{Buf, BufMut};
-
-        paste! {
-            $(
-                impl Index for $type {
-                    fn size() -> usize {
-                        std::mem::size_of::<$type>()
-                    }
-
-                    fn write(&self, mut buf: &mut [u8]) {
-                        buf.[< put_ $type>](*self)
-                    }
-
-                    fn read(mut buf: &[u8]) -> Self {
-                        buf.[< get_ $type>]()
-                    }
-                }
-            )*
-        }
-    };
-}
-
-for_all_primitives! { impl_index }
-
-pub struct WrappedNonNull<T>(pub NonNull<T>);
-
-unsafe impl<T> Send for WrappedNonNull<T> {}
-unsafe impl<T> Sync for WrappedNonNull<T> {}
-
-// TODO(MrCroxx): Add global error type.
-pub type Error = store::error::Error;
-
-pub type TinyLfuReadOnlyFileStoreCache<I, D> = container::Container<
-    I,
-    foyer_policy::eviction::tinylfu::TinyLfu<I>,
-    foyer_policy::eviction::tinylfu::Handle<I>,
-    D,
-    store::read_only_file_store::ReadOnlyFileStore<I, D>,
->;
-
-pub type TinyLfuReadOnlyFileStoreCacheConfig<I, D> = container::Config<
-    I,
-    foyer_policy::eviction::tinylfu::TinyLfu<I>,
-    foyer_policy::eviction::tinylfu::Handle<I>,
-    store::read_only_file_store::ReadOnlyFileStore<I, D>,
->;
-
-pub type LruReadOnlyFileStoreCache<I, D> = container::Container<
-    I,
-    foyer_policy::eviction::lru::Lru<I>,
-    foyer_policy::eviction::lru::Handle<I>,
-    D,
-    store::read_only_file_store::ReadOnlyFileStore<I, D>,
->;
-
-pub type LruReadOnlyFileStoreCacheConfig<I, D> = container::Config<
-    I,
-    foyer_policy::eviction::lru::Lru<I>,
-    foyer_policy::eviction::lru::Handle<I>,
-    store::read_only_file_store::ReadOnlyFileStore<I, D>,
->;
-
-pub use metrics::Metrics;
-
-pub use foyer_policy::eviction::{lru::Config as LruConfig, tinylfu::Config as TinyLfuConfig};
-pub use store::read_only_file_store::Config as ReadOnlyFileStoreConfig;
-
-#[cfg(test)]
-
-mod tests {
-    pub fn is_send_sync_static<T: Send + Sync + 'static>() {}
-}
diff --git a/foyer/src/store/error.rs b/foyer/src/store/error.rs
deleted file mode 100644
index c109bb55..00000000
--- a/foyer/src/store/error.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#[derive(thiserror::Error, Debug)]
-pub enum Error {
-    #[error("io error: {0}")]
-    Io(#[from] std::io::Error),
-    #[error("nix error: {0}")]
-    Nix(#[from] nix::errno::Errno),
-    #[error("other error: {0}")]
-    Other(String),
-}
-
-pub type Result<T> = core::result::Result<T, Error>;
diff --git a/foyer/src/store/file.rs b/foyer/src/store/file.rs
deleted file mode 100644
index d249b523..00000000
--- a/foyer/src/store/file.rs
+++ /dev/null
@@ -1,403 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use bytes::{Buf, BufMut};
-use nix::sys::{
-    stat::fstat,
-    uio::{pread, pwrite},
-};
-
-use super::{asyncify, error::Result};
-
-use std::{
-    fs::{remove_file, File, OpenOptions},
-    os::fd::{AsRawFd, RawFd},
-    path::{Path, PathBuf},
-    sync::atomic::{AtomicUsize, Ordering},
-};
-
-#[derive(Clone, Copy, Debug)]
-pub struct Location {
-    pub offset: u32,
-    pub len: u32,
-}
-
-impl Location {
-    pub fn write(&self, mut buf: &mut [u8]) {
-        buf.put_u32_le(self.offset);
-        buf.put_u32_le(self.len);
-    }
-
-    pub fn read(mut buf: &[u8]) -> Self {
-        let offset = buf.get_u32_le();
-        let len = buf.get_u32_le();
-        Self { offset, len }
-    }
-
-    pub fn size() -> usize {
-        8
-    }
-}
-
-// TODO(MrCroxx): Use buffer and direct I/O to better manage memory usage.
-pub struct AppendableFile {
-    path: PathBuf,
-
-    fd: RawFd,
-
-    size: AtomicUsize,
-
-    file: File,
-}
-
-impl AppendableFile {
-    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
-        let path = PathBuf::from(path.as_ref());
-
-        let opts = {
-            let mut opts = OpenOptions::new();
-            opts.create(true);
-            opts.write(true);
-            opts.read(true);
-            opts
-        };
-
-        let (file, fd, size) = asyncify({
-            let path = path.clone();
-            move || {
-                let file = opts.open(path)?;
-                let fd = file.as_raw_fd();
-                let stat = fstat(fd)?;
-                let size = stat.st_size as usize;
-                Ok((file, fd, size))
-            }
-        })
-        .await?;
-
-        Ok(Self {
-            path,
-            fd,
-            size: AtomicUsize::new(size),
-            file,
-        })
-    }
-
-    pub async fn append(&self, buf: Vec<u8>) -> Result<Location> {
-        let len = buf.len();
-        let offset = self.size.fetch_add(len, Ordering::Relaxed);
-        let fd = self.fd;
-        asyncify(move || {
-            pwrite(fd, &buf, offset as i64)?;
-            Ok(())
-        })
-        .await?;
-
-        Ok(Location {
-            offset: offset as u32,
-            len: len as u32,
-        })
-    }
-
-    #[allow(clippy::uninit_vec)]
-    pub async fn read(&self, offset: u64, len: usize) -> Result<Vec<u8>> {
-        let fd = self.fd;
-        let buf = asyncify(move || {
-            let mut buf = Vec::with_capacity(len);
-            unsafe { buf.set_len(len) };
-            pread(fd, &mut buf, offset as i64)?;
-            Ok(buf)
-        })
-        .await?;
-        Ok(buf)
-    }
-
-    pub fn len(&self) -> usize {
-        self.size.load(Ordering::Relaxed)
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    pub async fn size(&self) -> Result<usize> {
-        let fd = self.fd;
-        let size = asyncify(move || {
-            let stat = fstat(fd)?;
-            Ok(stat.st_size as usize)
-        })
-        .await?;
-        Ok(size)
-    }
-
-    pub async fn reclaim(self) -> Result<()> {
-        let file = self.file;
-        drop(file);
-
-        asyncify(move || {
-            remove_file(self.path)?;
-            Ok(())
-        })
-        .await
-    }
-}
-
-// TODO(MrCroxx): Use buffer and direct I/O to better manage memory usage.
-pub struct ReadableFile {
-    path: PathBuf,
-
-    fd: RawFd,
-
-    size: usize,
-
-    file: File,
-}
-
-impl ReadableFile {
-    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
-        let path = PathBuf::from(path.as_ref());
-
-        let opts = {
-            let mut opts = OpenOptions::new();
-            opts.read(true);
-            opts
-        };
-
-        let (file, fd, size) = asyncify({
-            let path = path.clone();
-            move || {
-                let file = opts.open(path)?;
-                let fd = file.as_raw_fd();
-                let stat = fstat(fd)?;
-                let size = stat.st_size as usize;
-                Ok((file, fd, size))
-            }
-        })
-        .await?;
-
-        Ok(Self {
-            path,
-            fd,
-            size,
-            file,
-        })
-    }
-
-    #[allow(clippy::uninit_vec)]
-    pub async fn read(&self, offset: u64, len: usize) -> Result<Vec<u8>> {
-        let fd = self.fd;
-        let buf = asyncify(move || {
-            let mut buf = Vec::with_capacity(len);
-            unsafe { buf.set_len(len) };
-            pread(fd, &mut buf, offset as i64)?;
-            Ok(buf)
-        })
-        .await?;
-        Ok(buf)
-    }
-
-    pub fn len(&self) -> usize {
-        self.size
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    pub async fn size(&self) -> Result<usize> {
-        let fd = self.fd;
-        let size = asyncify(move || {
-            let stat = fstat(fd)?;
-            Ok(stat.st_size as usize)
-        })
-        .await?;
-        Ok(size)
-    }
-
-    pub async fn reclaim(self) -> Result<()> {
-        let file = self.file;
-        drop(file);
-
-        asyncify(move || {
-            remove_file(self.path)?;
-            Ok(())
-        })
-        .await
-    }
-}
-
-pub struct WritableFile {
-    path: PathBuf,
-
-    fd: RawFd,
-
-    size: AtomicUsize,
-
-    file: File,
-}
-
-impl WritableFile {
-    pub async fn open(path: impl AsRef<Path>) -> Result<Self> {
-        let path = PathBuf::from(path.as_ref());
-
-        let opts = {
-            let mut opts = OpenOptions::new();
-            opts.create(true);
-            opts.write(true);
-            opts.read(true);
-            opts
-        };
-
-        let (file, fd, size) = asyncify({
-            let path = path.clone();
-            move || {
-                let file = opts.open(path)?;
-                let fd = file.as_raw_fd();
-                let stat = fstat(fd)?;
-                let size = stat.st_size as usize;
-                Ok((file, fd, size))
-            }
-        })
-        .await?;
-
-        Ok(Self {
-            path,
-            fd,
-            size: AtomicUsize::new(size),
-            file,
-        })
-    }
-
-    pub async fn append(&self, buf: Vec<u8>) -> Result<Location> {
-        let len = buf.len();
-        let offset = self.size.fetch_add(len, Ordering::Relaxed);
-        let fd = self.fd;
-        asyncify(move || {
-            pwrite(fd, &buf, offset as i64)?;
-            Ok(())
-        })
-        .await?;
-
-        Ok(Location {
-            offset: offset as u32,
-            len: len as u32,
-        })
-    }
-
-    /// # Safety
-    ///
-    /// Concurrent writes or appends on the same address are not guaranteed.
-    pub async fn write(&self, offset: u64, buf: Vec<u8>) -> Result<Location> {
-        let fd = self.fd;
-        let len = buf.len();
-        asyncify(move || {
-            pwrite(fd, &buf, offset as i64)?;
-            Ok(())
-        })
-        .await?;
-        Ok(Location {
-            offset: offset as u32,
-            len: len as u32,
-        })
-    }
-
-    #[allow(clippy::uninit_vec)]
-    pub async fn read(&self, offset: u64, len: usize) -> Result<Vec<u8>> {
-        let fd = self.fd;
-        let buf = asyncify(move || {
-            let mut buf = Vec::with_capacity(len);
-            unsafe { buf.set_len(len) };
-            pread(fd, &mut buf, offset as i64)?;
-            Ok(buf)
-        })
-        .await?;
-        Ok(buf)
-    }
-
-    pub async fn size(&self) -> Result<usize> {
-        let fd = self.fd;
-        let size = asyncify(move || {
-            let stat = fstat(fd)?;
-            Ok(stat.st_size as usize)
-        })
-        .await?;
-        Ok(size)
-    }
-
-    pub async fn reclaim(self) -> Result<()> {
-        let file = self.file;
-        drop(file);
-
-        asyncify(move || {
-            remove_file(self.path)?;
-            Ok(())
-        })
-        .await
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::tests::is_send_sync_static;
-
-    use super::*;
-
-    use futures::future::try_join_all;
-    use itertools::Itertools;
-    use tempfile::tempdir;
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn test_rw() {
-        is_send_sync_static::<AppendableFile>();
-        is_send_sync_static::<ReadableFile>();
-
-        let dir = tempdir().unwrap();
-        let path = PathBuf::from(dir.path()).join("testfile");
-
-        let afile = AppendableFile::open(&path).await.unwrap();
-
-        let bufs = (0..4).map(|i| vec![i as u8; 1024]).collect_vec();
-        let futures = bufs
-            .iter()
-            .map(|buf| afile.append(buf.clone()))
-            .collect_vec();
-        let locs = try_join_all(futures).await.unwrap();
-
-        assert_eq!(afile.len(), 4 * 1024);
-        assert_eq!(afile.size().await.unwrap(), 4 * 1024);
-        for (buf, loc) in bufs.iter().zip_eq(locs.iter()) {
-            assert_eq!(
-                buf,
-                &afile
-                    .read(loc.offset as u64, loc.len as usize)
-                    .await
-                    .unwrap()
-            );
-        }
-
-        drop(afile);
-
-        let rfile = ReadableFile::open(&path).await.unwrap();
-        for (buf, loc) in bufs.iter().zip_eq(locs.iter()) {
-            assert_eq!(
-                buf,
-                &rfile
-                    .read(loc.offset as u64, loc.len as usize)
-                    .await
-                    .unwrap()
-            );
-        }
-
-        drop(rfile);
-    }
-}
diff --git a/foyer/src/store/mod.rs b/foyer/src/store/mod.rs
deleted file mode 100644
index 079e73ce..00000000
--- a/foyer/src/store/mod.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-pub mod error;
-#[allow(dead_code)]
-pub mod file;
-pub mod read_only_file_store;
-
-use std::sync::Arc;
-
-use crate::{Data, Index, Metrics};
-use async_trait::async_trait;
-
-use error::Result;
-
-#[async_trait]
-pub trait Store: Send + Sync + Sized + 'static {
-    type I: Index;
-    type D: Data;
-    type C: Send + Sync + Clone + std::fmt::Debug + 'static;
-
-    async fn open(pool: usize, config: Self::C, metrics: Arc<Metrics>) -> Result<Self>;
-
-    async fn store(&self, index: Self::I, data: Self::D) -> Result<()>;
-
-    async fn load(&self, index: &Self::I) -> Result<Option<Self::D>>;
-
-    async fn delete(&self, index: &Self::I) -> Result<()>;
-}
-
-async fn asyncify<F, T>(f: F) -> error::Result<T>
-where
-    F: FnOnce() -> error::Result<T> + Send + 'static,
-    T: Send + 'static,
-{
-    #[cfg_attr(madsim, expect(deprecated))]
-    match tokio::task::spawn_blocking(f).await {
-        Ok(res) => res,
-        Err(_) => Err(error::Error::Other("background task failed".to_string())),
-    }
-}
-
-#[cfg(test)]
-pub mod tests {
-    use std::{collections::HashMap, sync::Arc};
-
-    use parking_lot::RwLock;
-
-    use super::error::Result;
-
-    use super::*;
-
-    #[derive(Clone, Debug, Default)]
-    pub struct MemoryStore<I: Index, D: Data + Clone> {
-        inner: Arc<RwLock<HashMap<I, D>>>,
-    }
-
-    impl<I: Index, D: Data + Clone> MemoryStore<I, D> {
-        pub fn new() -> Self {
-            Self {
-                inner: Arc::new(RwLock::new(HashMap::default())),
-            }
-        }
-    }
-
-    #[async_trait]
-    impl<I: Index, D: Data + Clone> Store for MemoryStore<I, D> {
-        type I = I;
-
-        type D = D;
-
-        type C = ();
-
-        async fn open(_pool: usize, _: Self::C, _metrics: Arc<Metrics>) -> Result<Self> {
-            Ok(Self::new())
-        }
-
-        async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {
-            let mut inner = self.inner.write();
-            inner.insert(index, data);
-            Ok(())
-        }
-
-        async fn load(&self, index: &Self::I) -> Result<Option<Self::D>> {
-            let inner = self.inner.read();
-            Ok(inner.get(index).cloned())
-        }
-
-        async fn delete(&self, index: &Self::I) -> Result<()> {
-            let mut inner = self.inner.write();
-            inner.remove(index);
-            Ok(())
-        }
-    }
-}
diff --git a/foyer/src/store/read_only_file_store.rs b/foyer/src/store/read_only_file_store.rs
deleted file mode 100644
index 0f113468..00000000
--- a/foyer/src/store/read_only_file_store.rs
+++ /dev/null
@@ -1,671 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{
-    fs::{create_dir_all, read_dir},
-    mem::swap,
-    path::{Path, PathBuf},
-};
-
-use std::{
-    collections::HashMap,
-    marker::PhantomData,
-    str::pattern::Pattern,
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc,
-    },
-};
-
-use async_trait::async_trait;
-
-use bytes::{Buf, BufMut};
-use itertools::Itertools;
-use rand::{thread_rng, Rng};
-use tokio::sync::{RwLock, RwLockWriteGuard};
-
-use crate::{Data, Index, Metrics};
-
-use super::{
-    asyncify,
-    error::Result,
-    file::{AppendableFile, Location, ReadableFile, WritableFile},
-    Store,
-};
-
-pub type FileId = u32;
-pub type SlotId = u32;
-
-const META_FILE_PREFIX: &str = "metafile-";
-const CACHE_FILE_PREFIX: &str = "cachefile-";
-
-#[derive(Clone, Debug)]
-pub struct Config {
-    /// path to store dir
-    pub dir: PathBuf,
-
-    /// store capacity
-    pub capacity: usize,
-
-    /// max cache file size
-    pub max_file_size: usize,
-
-    /// ratio of garbage to trigger reclaim
-    pub trigger_reclaim_garbage_ratio: f64,
-
-    /// ratio of size to trigger reclaim
-    pub trigger_reclaim_capacity_ratio: f64,
-
-    /// ratio of size to trigger randomly drop
-    pub trigger_random_drop_ratio: f64,
-
-    /// ratio of randomly dropped entries
-    pub random_drop_ratio: f64,
-
-    /// ratio of size to trigger write stall
-    ///
-    /// every new entry to insert will be dropped
-    pub write_stall_threshold_ratio: f64,
-}
-
-struct Frozen {
-    fid: FileId,
-    meta_file: WritableFile,
-    cache_file: ReadableFile,
-}
-
-struct Active {
-    fid: FileId,
-    meta_file: WritableFile,
-    cache_file: AppendableFile,
-}
-
-struct ReadOnlyFileStoreFiles {
-    active: Active,
-
-    frozens: HashMap<FileId, Frozen>,
-}
-
-/// `ReadOnlyFileStore` is a file store for read only entries.
-///
-/// The cache data for a cache key MUST NOT be updated.
-#[allow(clippy::type_complexity)]
-pub struct ReadOnlyFileStore<I, D>
-where
-    I: Index,
-    D: Data,
-{
-    pool: usize,
-
-    config: Arc<Config>,
-
-    indices: Arc<RwLock<HashMap<I, (FileId, SlotId, Location)>>>,
-
-    /// write lock is used when rotating active file or reclaiming frozen files
-    files: Arc<RwLock<ReadOnlyFileStoreFiles>>,
-
-    size: Arc<AtomicUsize>,
-
-    metrics: Arc<Metrics>,
-
-    _marker: PhantomData<D>,
-}
-
-impl<I, D> Clone for ReadOnlyFileStore<I, D>
-where
-    I: Index,
-    D: Data,
-{
-    fn clone(&self) -> Self {
-        Self {
-            pool: self.pool,
-            config: Arc::clone(&self.config),
-            indices: Arc::clone(&self.indices),
-            files: Arc::clone(&self.files),
-            size: Arc::clone(&self.size),
-            metrics: Arc::clone(&self.metrics),
-            _marker: PhantomData,
-        }
-    }
-}
-
-#[async_trait]
-impl<I, D> Store for ReadOnlyFileStore<I, D>
-where
-    I: Index,
-    D: Data,
-{
-    type I = I;
-
-    type D = D;
-
-    type C = Config;
-
-    async fn open(pool: usize, mut config: Self::C, metrics: Arc<Metrics>) -> Result<Self> {
-        config.dir = config.dir.join(format!("{:04}", pool));
-
-        let ids = asyncify({
-            let dir = config.dir.clone();
-            move || {
-                create_dir_all(&dir)?;
-
-                let mut ids: Vec<FileId> = read_dir(dir)?
-                    .map(|entry| entry.unwrap())
-                    .filter(|entry| {
-                        META_FILE_PREFIX.is_prefix_of(&entry.file_name().to_string_lossy())
-                    })
-                    .map(|entry| {
-                        entry.file_name().to_string_lossy()[META_FILE_PREFIX.len()..].to_string()
-                    })
-                    .map(|s| s.parse().unwrap())
-                    .collect_vec();
-                ids.sort();
-                Ok(ids)
-            }
-        })
-        .await?;
-
-        let mut frozens = HashMap::new();
-        let mut size = 0;
-        for id in &ids {
-            let frozen = Self::open_frozen_file(&config.dir, *id).await?;
-            // when restore, `len` is filled with `fstat(2)`
-            size += frozen.cache_file.len();
-            frozens.insert(*id, frozen);
-        }
-        // create new active file on every `open`
-        let id = ids.into_iter().max().unwrap_or(0) + 1;
-        let active = Self::open_active_file(&config.dir, id).await?;
-
-        let files = ReadOnlyFileStoreFiles { active, frozens };
-        let mut indices = HashMap::new();
-        for (fid, frozen) in &files.frozens {
-            Self::restore_meta(*fid, &frozen.meta_file, &mut indices).await?;
-        }
-
-        Ok(Self {
-            pool,
-            config: Arc::new(config),
-            indices: Arc::new(RwLock::new(indices)),
-            files: Arc::new(RwLock::new(files)),
-            size: Arc::new(AtomicUsize::new(size)),
-            metrics,
-            _marker: PhantomData,
-        })
-    }
-
-    #[allow(clippy::uninit_vec)]
-    async fn store(&self, index: Self::I, data: Self::D) -> Result<()> {
-        let _timer = self.metrics.latency_store.start_timer();
-
-        // append cache file and meta file
-        let (fid, sid, location) = {
-            let size_ratio = self.size.load(Ordering::Relaxed) as f64 / self.config.capacity as f64;
-            if self.config.write_stall_threshold_ratio > 0.0
-                && size_ratio >= self.config.write_stall_threshold_ratio
-            {
-                // write stall
-                return Ok(());
-            } else if self.config.trigger_random_drop_ratio > 0.0
-                && size_ratio >= self.config.trigger_random_drop_ratio
-            {
-                // random drop
-                let mut rng = thread_rng();
-                if rng.gen_range(0.0..1.0) < self.config.random_drop_ratio {
-                    return Ok(());
-                }
-            }
-
-            let buf = data.into();
-
-            let files = self.files.read().await;
-
-            let fid = files.active.fid;
-
-            let location = files.active.cache_file.append(buf).await?;
-
-            let mut buf = Vec::with_capacity(Self::meta_entry_size());
-
-            let tags = Tags { valid: true };
-
-            unsafe { buf.set_len(Self::meta_entry_size()) };
-            tags.write(&mut buf[..]);
-            index.write(&mut buf[Tags::size()..]);
-            location.write(&mut buf[Tags::size() + I::size()..]);
-
-            let Location {
-                offset: meta_offset,
-                len: _,
-            } = files.active.meta_file.append(buf).await?;
-            let sid = meta_offset / Self::meta_entry_size() as u32;
-
-            (fid, sid, location)
-        };
-
-        let active_file_size = (location.offset + location.len) as usize;
-
-        {
-            let mut indices = self.indices.write().await;
-            indices.insert(index, (fid, sid, location));
-            drop(indices);
-        }
-
-        let cache_data_size = self
-            .size
-            .fetch_add(location.len as usize, Ordering::Relaxed)
-            + location.len as usize;
-
-        self.metrics.bytes_store.inc_by(location.len as f64);
-        self.metrics.cache_data_size.set(cache_data_size as f64);
-
-        if active_file_size >= self.config.max_file_size {
-            let files = self.files.write().await;
-            // check size again in the critical section to prevent from double rotating
-            if files.active.cache_file.len() >= self.config.max_file_size {
-                self.rotate_active_file_locked(files).await?;
-            }
-        }
-
-        self.maybe_trigger_reclaim().await?;
-
-        Ok(())
-    }
-
-    async fn load(&self, index: &Self::I) -> Result<Option<Self::D>> {
-        let _timer = self.metrics.latency_load.start_timer();
-
-        // TODO(MrCroxx): add bloom filters ?
-        let (fid, _sid, location) = {
-            let indices = self.indices.read().await;
-
-            let (fid, sid, location) = match indices.get(index) {
-                Some((fid, sid, location)) => (*fid, *sid, *location),
-                None => return Ok(None),
-            };
-
-            (fid, sid, location)
-        };
-
-        let buf = {
-            let files = self.files.read().await;
-
-            if fid == files.active.fid {
-                files
-                    .active
-                    .cache_file
-                    .read(location.offset as u64, location.len as usize)
-                    .await?
-            } else {
-                match files.frozens.get(&fid) {
-                    Some(frozen) => {
-                        frozen
-                            .cache_file
-                            .read(location.offset as u64, location.len as usize)
-                            .await?
-                    }
-                    None => {
-                        tracing::error!("frozen file {} not found", fid);
-                        return Ok(None);
-                    }
-                }
-            }
-        };
-
-        self.metrics.bytes_load.inc_by(location.len as f64);
-
-        self.maybe_trigger_reclaim().await?;
-
-        Ok(Some(buf.into()))
-    }
-
-    async fn delete(&self, index: &Self::I) -> Result<()> {
-        let _timer = self.metrics.latency_delete.start_timer();
-
-        let (fid, sid, location) = {
-            let indices = self.indices.read().await;
-            let (fid, sid, location) = match indices.get(index) {
-                Some((fid, sid, location)) => (*fid, *sid, *location),
-                None => return Ok(()),
-            };
-            (fid, sid, location)
-        };
-
-        {
-            let empty_entry = vec![0; Self::meta_entry_size()];
-            let files = self.files.read().await;
-            if fid == files.active.fid {
-                files
-                    .active
-                    .meta_file
-                    .write(sid as u64 * Self::meta_entry_size() as u64, empty_entry)
-                    .await?;
-            } else {
-                match files.frozens.get(&fid) {
-                    Some(frozen) => {
-                        frozen
-                            .meta_file
-                            .write(sid as u64 * Self::meta_entry_size() as u64, empty_entry)
-                            .await?;
-                    }
-                    None => {
-                        tracing::error!("frozen file {} not found", fid);
-                    }
-                }
-            }
-        }
-
-        let cache_data_size = self
-            .size
-            .fetch_sub(location.len as usize, Ordering::Relaxed)
-            - location.len as usize;
-
-        self.metrics.bytes_delete.inc_by(location.len as f64);
-        self.metrics.cache_data_size.set(cache_data_size as f64);
-
-        self.maybe_trigger_reclaim().await?;
-
-        Ok(())
-    }
-}
-
-impl<I, D> ReadOnlyFileStore<I, D>
-where
-    I: Index,
-    D: Data,
-{
-    async fn rotate_active_file_locked<'a>(
-        &self,
-        mut files: RwLockWriteGuard<'a, ReadOnlyFileStoreFiles>,
-    ) -> Result<()> {
-        // rotate active file
-        let mut active = Self::open_active_file(&self.config.dir, files.active.fid + 1).await?;
-        swap(&mut active, &mut files.active);
-
-        // open frozen file
-        let id = active.fid;
-        let frozen = Self::open_frozen_file(&self.config.dir, id).await?;
-
-        // update frozen map
-        files.frozens.insert(id, frozen);
-
-        tracing::info!("active file rotated: {} => {}", id, id + 1);
-
-        Ok(())
-    }
-
-    async fn reclaim_frozen_file(&self, id: FileId) -> Result<()> {
-        tracing::info!("reclaiming frozen file {}", id);
-
-        let (fid, meta_file, cache_file) = {
-            let mut files = self.files.write().await;
-
-            let Frozen {
-                fid,
-                meta_file,
-                cache_file,
-            } = files.frozens.remove(&id).expect("frozen id not exists");
-
-            (fid, meta_file, cache_file)
-        };
-
-        let mut indices_to_delete = HashMap::new();
-        Self::restore_meta(fid, &meta_file, &mut indices_to_delete).await?;
-
-        let size = {
-            let mut size = 0;
-            let mut indices = self.indices.write().await;
-            for (index, (_fid, _sid, Location { offset: _, len })) in indices_to_delete {
-                indices.remove(&index);
-                size += len;
-            }
-            size as usize
-        };
-
-        let cache_data_size = self.size.fetch_sub(size, Ordering::Relaxed) - size;
-        self.metrics.cache_data_size.set(cache_data_size as f64);
-
-        meta_file.reclaim().await?;
-        cache_file.reclaim().await?;
-
-        tracing::info!("frozen file {} reclaimed", id);
-
-        Ok(())
-    }
-
-    async fn maybe_trigger_reclaim(&self) -> Result<()> {
-        // trigger by size ratio
-        if self.size.load(Ordering::Relaxed) as f64
-            >= self.config.capacity as f64 * self.config.trigger_reclaim_capacity_ratio
-        {
-            self.reclaim().await?;
-        }
-
-        // TODO(MrCroxx): trigger reclaim based on garbage ratio
-
-        Ok(())
-    }
-
-    /// Reclaim garbage to make room.
-    ///
-    /// Policy:
-    ///
-    /// [WIP] For now, simply reclaim the oldest frozen file.
-    ///
-    /// TODO(MrCroxx): better reclaim policy
-    async fn reclaim(&self) -> Result<()> {
-        let id = {
-            let files = self.files.read().await;
-            let id = files.frozens.keys().min();
-            match id {
-                Some(id) => *id,
-                None => return Ok(()),
-            }
-        };
-
-        self.reclaim_frozen_file(id).await?;
-
-        Ok(())
-    }
-
-    async fn open_active_file(dir: impl AsRef<Path>, id: FileId) -> Result<Active> {
-        let meta_file = WritableFile::open(Self::meta_file_path(&dir, id)).await?;
-        let cache_file = AppendableFile::open(Self::cache_file_path(&dir, id)).await?;
-        Ok(Active {
-            fid: id,
-            meta_file,
-            cache_file,
-        })
-    }
-
-    async fn open_frozen_file(dir: impl AsRef<Path>, id: FileId) -> Result<Frozen> {
-        let meta_file = WritableFile::open(Self::meta_file_path(&dir, id)).await?;
-        let cache_file = ReadableFile::open(Self::cache_file_path(&dir, id)).await?;
-
-        Ok(Frozen {
-            fid: id,
-            meta_file,
-            cache_file,
-        })
-    }
-
-    async fn restore_meta(
-        fid: FileId,
-        meta: &WritableFile,
-        indices: &mut HashMap<I, (FileId, SlotId, Location)>,
-    ) -> Result<()> {
-        let size = meta.size().await?;
-        let slots = size / Self::meta_entry_size();
-        let buf = meta.read(0, size).await?;
-        for sid in 0..slots {
-            let slice = &buf[sid * Self::meta_entry_size()..(sid + 1) * Self::meta_entry_size()];
-            let tags = Tags::read(slice);
-            if !tags.valid {
-                continue;
-            }
-            let index = I::read(&slice[Tags::size()..]);
-            let location = Location::read(&slice[Tags::size() + I::size()..]);
-            indices.insert(index, (fid, sid as SlotId, location));
-        }
-        Ok(())
-    }
-
-    fn cache_file_path(dir: impl AsRef<Path>, id: FileId) -> PathBuf {
-        PathBuf::from(dir.as_ref()).join(format!("{}{:08}", META_FILE_PREFIX, id))
-    }
-
-    fn meta_file_path(dir: impl AsRef<Path>, id: FileId) -> PathBuf {
-        PathBuf::from(dir.as_ref()).join(format!("{}{:08}", CACHE_FILE_PREFIX, id))
-    }
-
-    fn meta_entry_size() -> usize {
-        I::size() + Location::size() + Tags::size()
-    }
-}
-
-struct Tags {
-    valid: bool,
-}
-
-impl Tags {
-    fn size() -> usize {
-        1
-    }
-
-    fn write(&self, mut buf: &mut [u8]) {
-        let mut val = 0;
-        val |= self.valid as u8;
-        buf.put_u8(val);
-    }
-
-    fn read(mut buf: &[u8]) -> Self {
-        let val = buf.get_u8();
-        let valid = (val & 1) != 0;
-        Self { valid }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use tempfile::tempdir;
-
-    fn data(i: u8, len: usize) -> Vec<u8> {
-        vec![i; len]
-    }
-
-    #[tokio::test]
-    async fn test_read_only_file_store_simple() {
-        let dir = tempdir().unwrap();
-
-        let config = Config {
-            dir: dir.path().to_owned(),
-            max_file_size: 4 * 1024,
-            capacity: 16 * 1024,
-            trigger_reclaim_garbage_ratio: 0.0, // disabled
-            trigger_reclaim_capacity_ratio: 0.75,
-            trigger_random_drop_ratio: 0.0,   // disabled
-            random_drop_ratio: 0.0,           // disabled
-            write_stall_threshold_ratio: 0.0, // disabled
-        };
-
-        let store: ReadOnlyFileStore<u64, Vec<u8>> =
-            ReadOnlyFileStore::open(0, config, Arc::new(Metrics::default()))
-                .await
-                .unwrap();
-
-        store.store(1, data(1, 1024)).await.unwrap();
-        assert_eq!(store.load(&1).await.unwrap(), Some(data(1, 1024)));
-
-        store.store(2, data(2, 1024)).await.unwrap();
-        assert_eq!(store.load(&2).await.unwrap(), Some(data(2, 1024)));
-        store.store(3, data(3, 1024)).await.unwrap();
-        assert_eq!(store.load(&3).await.unwrap(), Some(data(3, 1024)));
-        store.store(4, data(4, 1024)).await.unwrap();
-        assert_eq!(store.load(&4).await.unwrap(), Some(data(4, 1024)));
-
-        // assert rotate
-        assert_eq!(store.files.read().await.frozens.len(), 1);
-
-        assert_eq!(store.load(&1).await.unwrap(), Some(data(1, 1024)));
-        assert_eq!(store.load(&2).await.unwrap(), Some(data(2, 1024)));
-        assert_eq!(store.load(&3).await.unwrap(), Some(data(3, 1024)));
-        assert_eq!(store.load(&4).await.unwrap(), Some(data(4, 1024)));
-
-        store.store(5, data(5, 4 * 1024)).await.unwrap();
-        assert_eq!(store.size.load(Ordering::Relaxed), 8 * 1024);
-        assert_eq!(store.files.read().await.frozens.len(), 2);
-
-        // assert reclaim
-        store.store(6, data(6, 4 * 1024)).await.unwrap();
-        assert_eq!(store.files.read().await.frozens.len(), 2);
-        assert_eq!(store.size.load(Ordering::Relaxed), 8 * 1024);
-
-        assert_eq!(store.load(&1).await.unwrap(), None);
-        assert_eq!(store.load(&2).await.unwrap(), None);
-        assert_eq!(store.load(&3).await.unwrap(), None);
-        assert_eq!(store.load(&4).await.unwrap(), None);
-
-        drop(dir);
-    }
-
-    #[tokio::test]
-    async fn test_read_only_file_store_recovery() {
-        let dir = tempdir().unwrap();
-
-        let config = Config {
-            dir: dir.path().to_owned(),
-            max_file_size: 4 * 1024,
-            capacity: 16 * 1024,
-            trigger_reclaim_garbage_ratio: 0.0, // disabled
-            trigger_reclaim_capacity_ratio: 0.75,
-            trigger_random_drop_ratio: 0.0,   // disabled
-            random_drop_ratio: 0.0,           // disabled
-            write_stall_threshold_ratio: 0.0, // disabled
-        };
-
-        let store: ReadOnlyFileStore<u64, Vec<u8>> =
-            ReadOnlyFileStore::open(0, config.clone(), Arc::new(Metrics::default()))
-                .await
-                .unwrap();
-
-        for i in 0..20 {
-            store.store(i, data(i as u8, 1024)).await.unwrap();
-        }
-
-        assert_eq!(store.files.read().await.frozens.len(), 2);
-        for i in 0..12 {
-            assert_eq!(store.load(&i).await.unwrap(), None);
-        }
-        for i in 12..20 {
-            assert_eq!(store.load(&i).await.unwrap(), Some(data(i as u8, 1024)));
-        }
-
-        drop(store);
-
-        let store: ReadOnlyFileStore<u64, Vec<u8>> =
-            ReadOnlyFileStore::open(0, config, Arc::new(Metrics::default()))
-                .await
-                .unwrap();
-
-        assert_eq!(store.files.read().await.frozens.len(), 3);
-        for i in 0..12 {
-            assert_eq!(store.load(&i).await.unwrap(), None);
-        }
-        for i in 12..20 {
-            assert_eq!(store.load(&i).await.unwrap(), Some(data(i as u8, 1024)));
-        }
-
-        drop(dir);
-    }
-}

From 85d8341bfe0d10f4f79070de38ecf974b9c04be6 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 30 Jun 2023 22:40:39 +0800
Subject: [PATCH 024/261] feat: foyer storage bench support flexible writers
 and readers (#31)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs | 106 ++++++++++++++++++++------------
 1 file changed, 65 insertions(+), 41 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index ecf0ca45..d53d7cb8 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -59,9 +59,6 @@ pub struct Args {
     #[arg(short, long, default_value_t = 60)]
     time: u64,
 
-    #[arg(long, default_value_t = 16)]
-    concurrency: usize,
-
     /// must be power of 2
     #[arg(long, default_value_t = 16)]
     pools: usize,
@@ -75,9 +72,11 @@ pub struct Args {
     #[arg(long, default_value = "")]
     iostat_dev: String,
 
+    /// (MiB)
     #[arg(long, default_value_t = 0.0)]
     w_rate: f64,
 
+    /// (MiB)
     #[arg(long, default_value_t = 0.0)]
     r_rate: f64,
 
@@ -106,6 +105,12 @@ pub struct Args {
 
     #[arg(long, default_value_t = 16 * 1024)]
     io_size: usize,
+
+    #[arg(long, default_value_t = 16)]
+    writers: usize,
+
+    #[arg(long, default_value_t = 16)]
+    readers: usize,
 }
 
 impl Args {
@@ -180,8 +185,7 @@ async fn main() {
     let store = TStore::open(config).await.unwrap();
 
     let (iostat_stop_tx, iostat_stop_rx) = oneshot::channel();
-    let (bench_stop_txs, bench_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
-        (0..args.concurrency).map(|_| oneshot::channel()).unzip();
+    let (bench_stop_tx, bench_stop_rx) = oneshot::channel();
 
     let handle_monitor = tokio::spawn({
         let iostat_path = iostat_path.clone();
@@ -196,19 +200,17 @@ async fn main() {
     });
     let handle_signal = tokio::spawn(async move {
         tokio::signal::ctrl_c().await.unwrap();
-        for bench_stop_tx in bench_stop_txs {
-            bench_stop_tx.send(()).unwrap();
-        }
+        bench_stop_tx.send(()).unwrap();
         iostat_stop_tx.send(()).unwrap();
     });
-    let handles = (bench_stop_rxs)
-        .into_iter()
-        .map(|stop| tokio::spawn(bench(args.clone(), store.clone(), metrics.clone(), stop)))
-        .collect_vec();
+    let handle_bench = tokio::spawn(bench(
+        args.clone(),
+        store.clone(),
+        metrics.clone(),
+        bench_stop_rx,
+    ));
 
-    for handle in handles {
-        handle.await.unwrap();
-    }
+    handle_bench.await.unwrap();
     handle_monitor.abort();
     handle_signal.abort();
 
@@ -228,46 +230,68 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::
     let w_rate = if args.w_rate == 0.0 {
         None
     } else {
-        Some(args.w_rate)
+        Some(args.w_rate * 1024.0 * 1024.0)
     };
     let r_rate = if args.r_rate == 0.0 {
         None
     } else {
-        Some(args.r_rate)
+        Some(args.r_rate * 1024.0 * 1024.0)
     };
 
     let index = Arc::new(AtomicU64::new(0));
 
-    let (w_stop_tx, w_stop_rx) = oneshot::channel();
-    let (r_stop_tx, r_stop_rx) = oneshot::channel();
+    let (w_stop_txs, w_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
+        (0..args.writers).map(|_| oneshot::channel()).unzip();
+    let (r_stop_txs, r_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
+        (0..args.readers).map(|_| oneshot::channel()).unzip();
+
+    println!("writers: {} readers: {}", args.writers, args.readers);
+
+    println!("txs: {:?}", w_stop_txs);
+
+    let w_handles = w_stop_rxs
+        .into_iter()
+        .map(|w_stop_rx| {
+            tokio::spawn(write(
+                args.entry_size,
+                w_rate,
+                index.clone(),
+                store.clone(),
+                args.time,
+                metrics.clone(),
+                w_stop_rx,
+            ))
+        })
+        .collect_vec();
+    let r_handles = r_stop_rxs
+        .into_iter()
+        .map(|r_stop_rx| {
+            tokio::spawn(read(
+                args.entry_size,
+                r_rate,
+                index.clone(),
+                store.clone(),
+                args.time,
+                metrics.clone(),
+                r_stop_rx,
+                args.lookup_range,
+            ))
+        })
+        .collect_vec();
 
-    let handle_w = tokio::spawn(write(
-        args.entry_size,
-        w_rate,
-        index.clone(),
-        store.clone(),
-        args.time,
-        metrics.clone(),
-        w_stop_rx,
-    ));
-    let handle_r = tokio::spawn(read(
-        args.entry_size,
-        r_rate,
-        index.clone(),
-        store.clone(),
-        args.time,
-        metrics.clone(),
-        r_stop_rx,
-        args.lookup_range,
-    ));
     tokio::spawn(async move {
         if let Ok(()) = stop.await {
-            let _ = w_stop_tx.send(());
-            let _ = r_stop_tx.send(());
+            for w_stop_tx in w_stop_txs {
+                let _ = w_stop_tx.send(());
+            }
+            for r_stop_tx in r_stop_txs {
+                let _ = r_stop_tx.send(());
+            }
         }
     });
 
-    join_all([handle_r, handle_w]).await;
+    join_all(w_handles).await;
+    join_all(r_handles).await;
 }
 
 #[allow(clippy::too_many_arguments)]

From 7c81272e2188ed1063a11a3adfc0c6ac099616e2 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 1 Jul 2023 20:19:16 +0800
Subject: [PATCH 025/261] feat: impl storage recovery (#32)

* feat: impl storage recovery

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock                          |  27 +++
 foyer-storage-bench/src/main.rs     |   8 +-
 foyer-storage/Cargo.toml            |   1 +
 foyer-storage/src/device/fs.rs      |  20 +-
 foyer-storage/src/device/mod.rs     |  19 +-
 foyer-storage/src/flusher.rs        |   2 +-
 foyer-storage/src/reclaimer.rs      |  13 +-
 foyer-storage/src/region.rs         |  78 +++++--
 foyer-storage/src/region_manager.rs |  55 +++--
 foyer-storage/src/store.rs          | 323 ++++++++++++++++++++++++----
 10 files changed, 443 insertions(+), 103 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 19376d0a..0d725ec4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -57,6 +57,17 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "async-channel"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"
+dependencies = [
+ "concurrent-queue",
+ "event-listener",
+ "futures-core",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.68"
@@ -179,6 +190,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
+[[package]]
+name = "concurrent-queue"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.3.2"
@@ -282,6 +302,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "event-listener"
+version = "2.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
+
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -365,6 +391,7 @@ dependencies = [
 name = "foyer-storage"
 version = "0.1.0"
 dependencies = [
+ "async-channel",
  "async-trait",
  "bitflags 2.3.3",
  "bytes",
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index d53d7cb8..5e5b21c0 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -111,6 +111,9 @@ pub struct Args {
 
     #[arg(long, default_value_t = 16)]
     readers: usize,
+
+    #[arg(long, default_value_t = 16)]
+    recover_concurrency: usize,
 }
 
 impl Args {
@@ -178,6 +181,7 @@ async fn main() {
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
         flushers: args.flushers,
         reclaimers: args.reclaimers,
+        recover_concurrency: args.recover_concurrency,
     };
 
     println!("{:#?}", config);
@@ -245,10 +249,6 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::
     let (r_stop_txs, r_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
         (0..args.readers).map(|_| oneshot::channel()).unzip();
 
-    println!("writers: {} readers: {}", args.writers, args.readers);
-
-    println!("txs: {:?}", w_stop_txs);
-
     let w_handles = w_stop_rxs
         .into_iter()
         .map(|w_stop_rx| {
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index b6a77967..f289b25b 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-channel = "1.8"
 async-trait = "0.1"
 bitflags = "2.3.1"
 bytes = "1"
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index fb54b40e..837bc927 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -91,18 +91,18 @@ impl Device for FsDevice {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> Result<()> {
+    ) -> Result<usize> {
         assert!(offset as usize + len <= self.inner.config.file_capacity);
 
         let fd = self.fd(region);
 
-        asyncify(move || {
-            nix::sys::uio::pwrite(fd, &buf.as_ref()[..len], offset as i64)?;
-            Ok(())
+        let res = asyncify(move || {
+            let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[..len], offset as i64)?;
+            Ok(res)
         })
         .await?;
 
-        Ok(())
+        Ok(res)
     }
 
     async fn read(
@@ -111,18 +111,18 @@ impl Device for FsDevice {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> Result<()> {
+    ) -> Result<usize> {
         assert!(offset as usize + len <= self.inner.config.file_capacity);
 
         let fd = self.fd(region);
 
-        asyncify(move || {
-            nix::sys::uio::pread(fd, &mut buf.as_mut()[..len], offset as i64)?;
-            Ok(())
+        let res = asyncify(move || {
+            let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[..len], offset as i64)?;
+            Ok(res)
         })
         .await?;
 
-        Ok(())
+        Ok(res)
     }
 
     async fn flush(&self) -> Result<()> {
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index a8d65270..6436e69d 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -34,8 +34,13 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
 
     async fn open(config: Self::Config) -> Result<Self>;
 
-    async fn write(&self, buf: impl IoBuf, region: RegionId, offset: u64, len: usize)
-        -> Result<()>;
+    async fn write(
+        &self,
+        buf: impl IoBuf,
+        region: RegionId,
+        offset: u64,
+        len: usize,
+    ) -> Result<usize>;
 
     async fn read(
         &self,
@@ -43,7 +48,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> Result<()>;
+    ) -> Result<usize>;
 
     async fn flush(&self) -> Result<()>;
 
@@ -108,8 +113,8 @@ pub mod tests {
             _region: RegionId,
             _offset: u64,
             _len: usize,
-        ) -> Result<()> {
-            Ok(())
+        ) -> Result<usize> {
+            Ok(0)
         }
 
         async fn read(
@@ -118,8 +123,8 @@ pub mod tests {
             _region: RegionId,
             _offset: u64,
             _len: usize,
-        ) -> Result<()> {
-            Ok(())
+        ) -> Result<usize> {
+            Ok(0)
         }
 
         async fn flush(&self) -> Result<()> {
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 76cc6b70..623ae792 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -200,7 +200,7 @@ where
                 // step 3: release buffer
                 self.buffers.release(buffer);
 
-                self.region_manager.post_flush(&region.id()).await;
+                self.region_manager.set_region_evictable(&region.id()).await;
 
                 tracing::info!("[flusher] finish flush task, region: {}", task.region_id);
             } else {
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 883de041..d921cfe9 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -24,6 +24,7 @@ use crate::{
     reinsertion::ReinsertionPolicy,
     store::Store,
 };
+use bytes::BufMut;
 use foyer_common::{
     code::{Key, Value},
     queue::AsyncQueue,
@@ -194,7 +195,17 @@ where
                 // step 2: do reinsertion
                 // TODO(MrCroxx): do reinsertion
 
-                // step 3: send clean region
+                // step 3: set region last block zero
+                let align = region.device().align();
+                let region_size = region.device().region_size();
+                let mut buf = region.device().io_buffer(align, align);
+                (&mut buf[..]).put_slice(&vec![0; align]);
+                region
+                    .device()
+                    .write(buf, task.region_id, (region_size - align) as u64, align)
+                    .await?;
+
+                // step 4: send clean region
                 self.clean_regions.release(task.region_id);
 
                 drop(guard);
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 4bdea8cb..154f3647 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -33,6 +33,20 @@ pub type RegionId = u32;
 /// 0 matches any version
 pub type Version = u32;
 
+pub enum AllocateResult {
+    Ok(WriteSlice),
+    Full { slice: WriteSlice, remain: usize },
+}
+
+impl AllocateResult {
+    pub fn unwrap(self) -> WriteSlice {
+        match self {
+            AllocateResult::Ok(slice) => slice,
+            AllocateResult::Full { .. } => unreachable!(),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct RegionInner<A>
 where
@@ -51,7 +65,7 @@ where
     wakers: Vec<Waker>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Region<A, D>
 where
     A: BufferAllocator,
@@ -102,7 +116,7 @@ where
         }
     }
 
-    pub fn allocate(&self, size: usize) -> Option<WriteSlice> {
+    pub fn allocate(&self, size: usize) -> AllocateResult {
         let callback = {
             let inner = self.inner.clone();
             move || {
@@ -114,28 +128,44 @@ where
 
         let mut inner = self.inner.write();
 
-        if inner.len + size > inner.capacity {
-            return None;
-        }
-
         inner.writers += 1;
         let version = inner.version;
         let offset = inner.len;
-        inner.len += size;
-        let buffer = inner.buffer.as_mut().unwrap();
-
-        let slice = unsafe { SliceMut::new(&mut buffer[offset..offset + size]) };
         let region_id = self.id;
 
-        let slice = WriteSlice {
-            slice,
-            region_id,
-            version,
-            offset,
-            callback: Box::new(callback),
-        };
-
-        Some(slice)
+        // reserve 1 align size for region footer
+        if inner.len + size + self.device.align() > inner.capacity {
+            // if full, return the reserved 1 aligen write buf
+            let remain = self.device.region_size() - inner.len;
+            inner.len = self.device.region_size();
+            let range = inner.len - self.device.align()..inner.len;
+
+            let buffer = inner.buffer.as_mut().unwrap();
+            let slice = unsafe { SliceMut::new(&mut buffer[range]) };
+
+            let slice = WriteSlice {
+                slice,
+                region_id,
+                version,
+                offset,
+                callback: Box::new(callback),
+            };
+            AllocateResult::Full { slice, remain }
+        } else {
+            inner.len += size;
+
+            let buffer = inner.buffer.as_mut().unwrap();
+            let slice = unsafe { SliceMut::new(&mut buffer[offset..offset + size]) };
+
+            let slice = WriteSlice {
+                slice,
+                region_id,
+                version,
+                offset,
+                callback: Box::new(callback),
+            };
+            AllocateResult::Ok(slice)
+        }
     }
 
     pub async fn load(
@@ -200,9 +230,15 @@ where
                 start + offset + len
             );
             let s = unsafe { SliceMut::new(&mut buf[offset..offset + len]) };
-            self.device
+            if self
+                .device
                 .read(s, region, (start + offset) as u64, len)
-                .await?;
+                .await?
+                != len
+            {
+                self.inner.write().physical_readers -= 1;
+                return Ok(None);
+            }
             offset += len;
         }
 
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index b0770269..91ac7d87 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -26,7 +26,7 @@ use crate::{
     device::{BufferAllocator, Device},
     flusher::{FlushTask, Flusher},
     reclaimer::{ReclaimTask, Reclaimer},
-    region::{Region, RegionId, WriteSlice},
+    region::{AllocateResult, Region, RegionId},
 };
 
 #[derive(Debug)]
@@ -41,17 +41,13 @@ where
 intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEpItem<L> { link: L } where L: Link }
 key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
 
-struct RegionManagerInner<A, E, EL>
+struct RegionManagerInner<E, EL>
 where
-    A: BufferAllocator,
     E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
     EL: Link,
 {
     current: Option<RegionId>,
 
-    buffers: Arc<AsyncQueue<Vec<u8, A>>>,
-    clean_regions: Arc<AsyncQueue<RegionId>>,
-
     eviction: E,
 
     _marker: PhantomData<EL>,
@@ -64,7 +60,10 @@ where
     E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
     EL: Link,
 {
-    inner: Arc<RwLock<RegionManagerInner<A, E, EL>>>,
+    inner: Arc<RwLock<RegionManagerInner<E, EL>>>,
+
+    buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+    clean_regions: Arc<AsyncQueue<RegionId>>,
 
     regions: Vec<Region<A, D>>,
     items: Vec<Arc<RegionEpItem<EL>>>,
@@ -107,14 +106,15 @@ where
 
         let inner = RegionManagerInner {
             current: None,
-            buffers,
-            clean_regions,
+
             eviction,
             _marker: PhantomData,
         };
 
         Self {
             inner: Arc::new(RwLock::new(inner)),
+            buffers,
+            clean_regions,
             regions,
             items,
             flusher,
@@ -123,25 +123,27 @@ where
     }
 
     /// Allocate a buffer slice with given size in an active region to write.
-    pub async fn allocate(&self, size: usize) -> WriteSlice {
+    pub async fn allocate(&self, size: usize) -> AllocateResult {
         let mut inner = self.inner.write().await;
 
         // try allocate from current region
         if let Some(region_id) = inner.current {
             let region = self.region(&region_id);
-            if let Some(slice) = region.allocate(size) {
-                return slice;
+            match region.allocate(size) {
+                AllocateResult::Ok(slice) => return AllocateResult::Ok(slice),
+                AllocateResult::Full { slice, remain } => {
+                    // current region is full, schedule flushing
+                    self.flusher.submit(FlushTask { region_id }).await.unwrap();
+                    inner.current = None;
+                    return AllocateResult::Full { slice, remain };
+                }
             }
-
-            // current region is full, schedule flushing
-            self.flusher.submit(FlushTask { region_id }).await.unwrap();
-            inner.current = None;
         }
 
         assert!(inner.current.is_none());
 
-        tracing::debug!("clean regions: {}", inner.clean_regions.len());
-        if inner.clean_regions.len() < self.reclaimer.runners() {
+        tracing::debug!("clean regions: {}", self.clean_regions.len());
+        if self.clean_regions.len() < self.reclaimer.runners() {
             if let Some(item) = inner.eviction.pop() {
                 self.reclaimer
                     .submit(ReclaimTask { region_id: item.id })
@@ -150,11 +152,11 @@ where
             }
         }
 
-        let region_id = inner.clean_regions.acquire().await;
+        let region_id = self.clean_regions.acquire().await;
         tracing::info!("switch to clean region: {}", region_id);
 
         let region = self.region(&region_id);
-        let buffer = inner.buffers.acquire().await;
+        let buffer = self.buffers.acquire().await;
         region.attach_buffer(buffer);
 
         let slice = region.allocate(size).unwrap();
@@ -162,7 +164,7 @@ where
         region.advance();
         inner.current = Some(region_id);
 
-        slice
+        AllocateResult::Ok(slice)
     }
 
     pub fn region(&self, id: &RegionId) -> &Region<A, D> {
@@ -177,8 +179,15 @@ where
         }
     }
 
-    pub async fn post_flush(&self, id: &RegionId) {
+    pub async fn set_region_evictable(&self, id: &RegionId) {
         let mut inner = self.inner.write().await;
-        inner.eviction.push(self.items[*id as usize].clone());
+        let item = &self.items[*id as usize];
+        if !item.link.is_linked() {
+            inner.eviction.push(item.clone());
+        }
+    }
+
+    pub fn clean_regions(&self) -> &AsyncQueue<RegionId> {
+        &self.clean_regions
     }
 }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index cde2ac3e..cc7d2115 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -15,7 +15,7 @@
 use std::{fmt::Debug, marker::PhantomData, sync::Arc};
 
 use bytes::{Buf, BufMut};
-use foyer_common::{bits::align_up, queue::AsyncQueue};
+use foyer_common::{bits, queue::AsyncQueue};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use twox_hash::XxHash64;
 
@@ -26,13 +26,15 @@ use crate::{
     flusher::Flusher,
     indices::{Index, Indices},
     reclaimer::Reclaimer,
-    region::RegionId,
+    region::{Region, RegionId},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
 };
 use foyer_common::code::{Key, Value};
 use std::hash::Hasher;
 
+const REGION_MAGIC: u64 = 0x19970327;
+
 pub struct StoreConfig<D, AP, EP, RP, EL>
 where
     D: Device,
@@ -48,6 +50,7 @@ where
     pub buffer_pool_size: usize,
     pub flushers: usize,
     pub reclaimers: usize,
+    pub recover_concurrency: usize,
 }
 
 impl<D, AP, EP, RP, EL> Debug for StoreConfig<D, AP, EP, RP, EL>
@@ -105,6 +108,8 @@ where
     EL: Link,
 {
     pub async fn open(config: StoreConfig<D, AP, EP, RP, EL>) -> Result<Arc<Self>> {
+        tracing::info!("open store with config:\n{:#?}", config);
+
         let device = D::open(config.device_config).await?;
 
         let buffers = Arc::new(AsyncQueue::new());
@@ -115,9 +120,6 @@ where
         }
 
         let clean_regions = Arc::new(AsyncQueue::new());
-        for region_id in 0..device.regions() as RegionId {
-            clean_regions.release(region_id);
-        }
 
         let flusher = Arc::new(Flusher::new(config.flushers));
         let reclaimer = Arc::new(Reclaimer::new(config.reclaimers));
@@ -137,11 +139,13 @@ where
         let store = Arc::new(Self {
             indices: indices.clone(),
             region_manager: region_manager.clone(),
-            device,
+            device: device.clone(),
             admission: config.admission,
             _marker: PhantomData,
         });
 
+        store.recover(config.recover_concurrency).await?;
+
         flusher.run(buffers, region_manager.clone()).await;
         reclaimer
             .run(
@@ -163,23 +167,20 @@ where
 
         let serialized_len = self.serialized_len(&key, &value);
 
-        let mut slice = self.region_manager.allocate(serialized_len).await;
-
-        let mut offset = 0;
-        value.write(&mut slice.as_mut()[offset..offset + value.serialized_len()]);
-        offset += value.serialized_len();
-        key.write(&mut slice.as_mut()[offset..offset + key.serialized_len()]);
-        offset += key.serialized_len();
-
-        let checksum = checksum(&slice.as_ref()[..offset]);
-
-        let footer = Footer {
-            key_len: key.serialized_len() as u32,
-            value_len: value.serialized_len() as u32,
-            checksum,
+        let mut slice = match self.region_manager.allocate(serialized_len).await {
+            crate::region::AllocateResult::Ok(slice) => slice,
+            crate::region::AllocateResult::Full { mut slice, remain } => {
+                // current region is full, write region footer and try allocate again
+                let footer = RegionFooter {
+                    magic: REGION_MAGIC,
+                    padding: remain as u64,
+                };
+                footer.write(slice.as_mut());
+                self.region_manager.allocate(serialized_len).await.unwrap()
+            }
         };
-        offset = slice.len() - Footer::serialized_len();
-        footer.write(&mut slice.as_mut()[offset..]);
+
+        write_entry(slice.as_mut(), &key, &value);
 
         let index = Index {
             region: slice.region_id(),
@@ -208,22 +209,18 @@ where
         self.region_manager.record_access(&index.region).await;
         let region = self.region_manager.region(&index.region);
         let start = index.offset as usize;
-        // TODO(MrCroxx): read value and checksum only
         let end = start + index.len as usize;
+
+        // TODO(MrCroxx): read value only
         let slice = match region.load(start..end, index.version).await? {
             Some(slice) => slice,
             None => return Ok(None),
         };
 
-        let checksum = checksum(&slice.as_ref()[..(index.value_len + index.key_len) as usize]);
-        let expected = (&slice.as_ref()[slice.len() - 8..]).get_u64();
-        if checksum != expected {
-            return Err(Error::ChecksumMismatch { checksum, expected });
+        match read_entry::<K, V>(slice.as_ref()) {
+            Some((_key, value)) => Ok(Some(value)),
+            None => Ok(None),
         }
-
-        let value = V::read(&slice.as_ref()[..index.value_len as usize]);
-
-        Ok(Some(value))
     }
 
     pub fn remove(&self, key: &K) {
@@ -231,25 +228,82 @@ where
     }
 
     fn serialized_len(&self, key: &K, value: &V) -> usize {
-        let unaligned = key.serialized_len() + value.serialized_len() + Footer::serialized_len();
-        align_up(self.device.align(), unaligned)
+        let unaligned =
+            key.serialized_len() + value.serialized_len() + EntryFooter::serialized_len();
+        bits::align_up(self.device.align(), unaligned)
+    }
+
+    async fn recover(&self, concurrency: usize) -> Result<()> {
+        tracing::info!("start store recovery");
+
+        let (tx, rx) = async_channel::bounded(concurrency);
+
+        let mut handles = vec![];
+        for region_id in 0..self.device.regions() as RegionId {
+            let itx = tx.clone();
+            let irx = rx.clone();
+            let region_manager = self.region_manager.clone();
+            let indices = self.indices.clone();
+            let handle = tokio::spawn(async move {
+                itx.send(()).await.unwrap();
+                let res = Self::recover_region(region_id, region_manager, indices).await;
+                irx.recv().await.unwrap();
+                res
+            });
+            handles.push(handle);
+        }
+
+        let mut recovered = 0;
+        for (region_id, handle) in handles.into_iter().enumerate() {
+            if handle.await.map_err(Error::other)?? {
+                tracing::debug!("region {} is recovered", region_id);
+                recovered += 1;
+            }
+        }
+
+        tracing::info!("finish store recovery, {} region recovered", recovered);
+
+        Ok(())
+    }
+
+    /// return `true` if region is valid, otherwise `false`
+    async fn recover_region(
+        region_id: RegionId,
+        region_manager: Arc<RegionManager<BA, D, EP, EL>>,
+        indices: Arc<Indices<K>>,
+    ) -> Result<bool> {
+        let region = region_manager.region(&region_id).clone();
+        let res = if let Some(mut iter) = RegionEntryIter::<K, V, BA, D>::open(region).await? {
+            while let Some(index) = iter.next().await? {
+                indices.insert(index);
+            }
+            region_manager.set_region_evictable(&region_id).await;
+            true
+        } else {
+            region_manager.clean_regions().release(region_id);
+            false
+        };
+        Ok(res)
     }
 }
 
-struct Footer {
+#[derive(Debug)]
+struct EntryFooter {
     key_len: u32,
     value_len: u32,
+    padding: u32,
     checksum: u64,
 }
 
-impl Footer {
+impl EntryFooter {
     fn serialized_len() -> usize {
-        4 + 4 + 8
+        4 + 4 + 4 + 8
     }
 
     fn write(&self, mut buf: &mut [u8]) {
         buf.put_u32(self.key_len);
         buf.put_u32(self.value_len);
+        buf.put_u32(self.padding);
         buf.put_u64(self.checksum);
     }
 
@@ -257,18 +311,215 @@ impl Footer {
     fn read(mut buf: &[u8]) -> Self {
         let key_len = buf.get_u32();
         let value_len = buf.get_u32();
+        let padding = buf.get_u32();
         let checksum = buf.get_u64();
 
         Self {
             key_len,
             value_len,
+            padding,
             checksum,
         }
     }
 }
 
+#[derive(Debug)]
+struct RegionFooter {
+    /// magic number to decide a valid region
+    magic: u64,
+
+    /// padding from the end of the last entry footer to the end of region
+    padding: u64,
+}
+
+impl RegionFooter {
+    fn write(&self, buf: &mut [u8]) {
+        let mut offset = buf.len();
+
+        offset -= 8;
+        (&mut buf[offset..offset + 8]).put_u64(self.magic);
+
+        offset -= 8;
+        (&mut buf[offset..offset + 8]).put_u64(self.padding);
+    }
+
+    fn read(buf: &[u8]) -> Self {
+        let mut offset = buf.len();
+
+        offset -= 8;
+        let magic = (&buf[offset..offset + 8]).get_u64();
+
+        offset -= 8;
+        let padding = (&buf[offset..offset + 8]).get_u64();
+
+        Self { magic, padding }
+    }
+}
+
+/// | value | key | <padding> | footer |
+///
+/// # Safety
+///
+/// `buf.len()` must excatly fit entry size
+fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V)
+where
+    K: Key,
+    V: Value,
+{
+    let mut offset = 0;
+    value.write(&mut buf[offset..offset + value.serialized_len()]);
+    offset += value.serialized_len();
+    key.write(&mut buf[offset..offset + key.serialized_len()]);
+    offset += key.serialized_len();
+
+    let checksum = checksum(&buf[..offset]);
+    let padding = buf.len() as u32
+        - key.serialized_len() as u32
+        - value.serialized_len() as u32
+        - EntryFooter::serialized_len() as u32;
+
+    let footer = EntryFooter {
+        key_len: key.serialized_len() as u32,
+        value_len: value.serialized_len() as u32,
+        padding,
+        checksum,
+    };
+    offset = buf.len() - EntryFooter::serialized_len();
+    footer.write(&mut buf[offset..]);
+}
+
+/// | value | key | <padding> | footer |
+///
+/// # Safety
+///
+/// `buf.len()` must excatly fit entry size
+fn read_entry<K, V>(buf: &[u8]) -> Option<(K, V)>
+where
+    K: Key,
+    V: Value,
+{
+    let mut offset = buf.len();
+
+    offset -= EntryFooter::serialized_len();
+    let footer = EntryFooter::read(&buf[offset..]);
+
+    offset = 0;
+    let value = V::read(&buf[offset..offset + footer.value_len as usize]);
+
+    offset += footer.value_len as usize;
+    let key = K::read(&buf[offset..offset + footer.key_len as usize]);
+
+    offset += footer.key_len as usize;
+    let checksum = checksum(&buf[..offset]);
+    if checksum != footer.checksum {
+        tracing::warn!(
+            "read entry error: {}",
+            Error::ChecksumMismatch {
+                checksum,
+                expected: footer.checksum,
+            }
+        );
+        return None;
+    }
+
+    Some((key, value))
+}
+
 fn checksum(buf: &[u8]) -> u64 {
     let mut hasher = XxHash64::with_seed(0);
     hasher.write(buf);
     hasher.finish()
 }
+
+struct RegionEntryIter<K, V, BA, D>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+{
+    region: Region<BA, D>,
+
+    cursor: usize,
+
+    _marker: PhantomData<(K, V)>,
+}
+
+impl<K, V, BA, D> RegionEntryIter<K, V, BA, D>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+{
+    async fn open(region: Region<BA, D>) -> Result<Option<Self>> {
+        let region_size = region.device().region_size();
+        let align = region.device().align();
+
+        let slice = match region.load(region_size - align..region_size, 0).await? {
+            Some(slice) => slice,
+            None => return Ok(None),
+        };
+
+        let footer = RegionFooter::read(slice.as_ref());
+        if footer.magic != REGION_MAGIC {
+            return Ok(None);
+        }
+        let cursor = region_size - footer.padding as usize;
+        Ok(Some(Self {
+            region,
+            cursor,
+            _marker: PhantomData,
+        }))
+    }
+
+    async fn next(&mut self) -> Result<Option<Index<K>>> {
+        if self.cursor == 0 {
+            return Ok(None);
+        }
+
+        let align = self.region.device().align();
+
+        let slice = self
+            .region
+            .load(self.cursor - align..self.cursor, 0)
+            .await?
+            .unwrap();
+
+        let footer =
+            EntryFooter::read(&slice.as_ref()[align - EntryFooter::serialized_len()..align]);
+        let entry_len = (footer.value_len + footer.key_len + footer.padding) as usize
+            + EntryFooter::serialized_len();
+
+        let abs_start = self.cursor - entry_len + footer.value_len as usize;
+        let abs_end = self.cursor - entry_len + (footer.value_len + footer.key_len) as usize;
+        let align_start = bits::align_down(align, abs_start);
+        let align_end = bits::align_up(align, abs_end);
+
+        let key = if align_start == self.cursor - align && align_end == self.cursor {
+            // key and foooter in the same block, read directly from slice
+            let rel_start =
+                align - EntryFooter::serialized_len() - (footer.padding + footer.key_len) as usize;
+            let rel_end = align - EntryFooter::serialized_len() - footer.padding as usize;
+            K::read(&slice.as_ref()[rel_start..rel_end])
+        } else {
+            let slice = self.region.load(align_start..align_end, 0).await?.unwrap();
+            let rel_start = abs_start - align_start;
+            let rel_end = abs_end - align_start;
+
+            K::read(&slice.as_ref()[rel_start..rel_end])
+        };
+
+        self.cursor -= entry_len;
+
+        Ok(Some(Index {
+            key,
+            region: self.region.id(),
+            version: 0,
+            offset: self.cursor as u32,
+            len: entry_len as u32,
+            key_len: footer.key_len,
+            value_len: footer.value_len,
+        }))
+    }
+}

From ad3f1255f43732223b8457e7dc21b6a13f8e43b5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 3 Jul 2023 15:38:12 +0800
Subject: [PATCH 026/261] feat: introduce segment fifo eviction policy (#35)

* feat: add segment fifo eviction policy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* export fifo fs store

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/core/adapter.rs  |  88 +++++-
 foyer-intrusive/src/eviction/fifo.rs | 444 +++++++++++++++++++++++++++
 foyer-intrusive/src/eviction/lfu.rs  |  33 +-
 foyer-intrusive/src/eviction/lru.rs  |  29 +-
 foyer-intrusive/src/eviction/mod.rs  |   5 +-
 foyer-storage/src/lib.rs             |  13 +
 6 files changed, 569 insertions(+), 43 deletions(-)
 create mode 100644 foyer-intrusive/src/eviction/fifo.rs

diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index cbc24ed2..c41b1bf0 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -69,6 +69,23 @@ pub unsafe trait KeyAdapter: Adapter {
     ) -> *const Self::Key;
 }
 
+/// # Safety
+///
+/// Pointer operations MUST be valid.
+///
+/// [`PriorityAdapter`] is recommanded to be generated by macro `priority_adapter!`.
+pub unsafe trait PriorityAdapter: Adapter {
+    type Priority: PartialEq + Eq + PartialOrd + Ord + Clone + Copy + Into<usize>;
+
+    /// # Safety
+    ///
+    /// Pointer operations MUST be valid.
+    unsafe fn item2priority(
+        &self,
+        item: *const <Self::PointerOps as PointerOps>::Item,
+    ) -> *const Self::Priority;
+}
+
 /// Macro to generate an implementation of [`Adapter`] for instrusive container and items.
 ///
 /// The basic syntax to create an adapter is:
@@ -234,14 +251,75 @@ macro_rules! key_adapter {
     };
 }
 
-macro_rules! t {
+/// Macro to generate an implementation of [`PriorityAdapter`] for instrusive container and items.
+/// ///
+/// The basic syntax to create an adapter is:
+///
+/// ```rust,ignore
+/// priority_adapter! { Adapter = Item { priority_field: PriorityType } }
+/// ```
+///
+/// # Generics
+///
+/// This macro supports generic arguments:
+///
+/// Note that due to macro parsing limitations, `T: Trait` bounds are not
+/// supported in the generic argument list. You must list any trait bounds in
+/// a separate `where` clause at the end of the macro.
+///
+/// # Examples
+///
+/// ```
+/// use foyer_intrusive::{intrusive_adapter, priority_adapter};
+/// use foyer_intrusive::core::adapter::{Adapter, PriorityAdapter, Link};
+/// use foyer_intrusive::core::pointer::PointerOps;
+/// use foyer_intrusive::eviction::EvictionPolicy;
+/// use std::sync::Arc;
+///
+/// #[derive(Debug)]
+/// pub struct Item<L>
+/// where
+///     L: Link
+/// {
+///     link: L,
+///     priority: usize,
+/// }
+///
+/// intrusive_adapter! { ItemAdapter<L> = Arc<Item<L>>: Item<L> { link: L} where L: Link }
+/// priority_adapter! { ItemAdapter<L> = Item<L> { priority: usize } where L: Link }
+/// ```
+#[macro_export]
+macro_rules! priority_adapter {
+    (@impl
+        $adapter:ident ($($args:tt),*) = $item:ty { $field:ident: $priority:ty } $($where_:tt)*
+    ) => {
+        unsafe impl<$($args),*> $crate::core::adapter::PriorityAdapter for $adapter<$($args),*> $($where_)*{
+            type Priority = $priority;
+
+            unsafe fn item2priority(
+                &self,
+                item: *const <Self::PointerOps as PointerOps>::Item,
+            ) -> *const Self::Priority {
+                (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
+            }
+        }
+    };
+    (
+        $name:ident = $($rest:tt)*
+    ) => {
+        priority_adapter! {@impl
+            $name () = $($rest)*
+        }
+    };
     (
-        <$($args:tt),*>
-    ) => {};
+        $name:ident<$($args:tt),*> = $($rest:tt)*
+    ) => {
+        priority_adapter! {@impl
+            $name ($($args),*) = $($rest)*
+        }
+    };
 }
 
-t! { <A, B, C> }
-
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
new file mode 100644
index 00000000..86d3d97d
--- /dev/null
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -0,0 +1,444 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{mem::ManuallyDrop, ptr::NonNull};
+
+use itertools::Itertools;
+
+use crate::{
+    collections::dlist::{DList, DListIter, DListLink},
+    core::{
+        adapter::{Adapter, Link, PriorityAdapter},
+        pointer::PointerOps,
+    },
+    intrusive_adapter,
+};
+
+use super::EvictionPolicy;
+
+#[derive(Debug, Clone)]
+pub struct FifoConfig {
+    /// `segment_ratios` is used to compute the ratio of each segment's size.
+    ///
+    /// The formula is as follows:
+    ///
+    /// `segment's size = total_segments * (segment's ratio / sum(ratio))`
+    segment_ratios: Vec<usize>,
+}
+
+#[derive(Debug, Default)]
+pub struct FifoLink {
+    link_queue: DListLink,
+
+    priority: usize,
+}
+
+impl FifoLink {
+    fn raw(&self) -> NonNull<Self> {
+        unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
+    }
+}
+
+impl Link for FifoLink {
+    fn is_linked(&self) -> bool {
+        self.link_queue.is_linked()
+    }
+}
+
+intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link_queue: DListLink } }
+
+/// Segmented FIFO policy
+///
+/// It divides the fifo queue into N segments. Each segment holds
+/// number of items proportional to its segment ratio. For example,
+/// if we have 3 segments and the ratio of [2, 1, 1], the lowest
+/// priority segment will hold 50% of the items whereas the other
+/// two higher priority segments will hold 25% each.
+///
+/// On insertion, a priority is used as an Insertion Point. E.g. a pri-2
+/// region will be inserted into the third highest priority segment. After
+/// the insertion is completed, we will trigger rebalance, where this
+/// region may be moved to below the insertion point, if the segment it
+/// was originally inserted into had exceeded the size allowed by its ratio.
+///
+/// Our rebalancing scheme allows the lowest priority segment to grow beyond
+/// its ratio allows for, since there is no lower segment to move into.
+///
+/// Also note that rebalancing is also triggered after an eviction.
+///
+/// The rate of inserting new regions look like the following:
+/// Pri-2 ---
+///          \
+/// Pri-1 -------
+///              \
+/// Pri-0 ------------
+/// When pri-2 exceeds its ratio, it effectively downgrades the oldest region in
+/// pri-2 to pri-1, and that region is now pushed down at the combined rate of
+/// (new pri-1 regions + new pri-2 regions), so effectively it gets evicted out
+/// of the system faster once it is beyond the pri-2 segment ratio. Segment
+/// ratio is put in place to prevent the lower segments getting so small a
+/// portion of the flash device.
+pub struct Fifo<A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    // Note: All queue share the same dlist link.
+    segments: Vec<DList<FifoLinkAdapter>>,
+
+    config: FifoConfig,
+    total_ratio: usize,
+
+    adapter: A,
+}
+
+impl<A> Drop for Fifo<A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    fn drop(&mut self) {
+        let mut to_remove = vec![];
+        for ptr in self.iter() {
+            to_remove.push(ptr.clone());
+        }
+        for ptr in to_remove {
+            self.remove(&ptr);
+        }
+    }
+}
+
+impl<A> Fifo<A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    pub fn new(config: FifoConfig) -> Self {
+        let segments = (0..config.segment_ratios.len())
+            .map(|_| DList::new())
+            .collect_vec();
+        let total_ratio = config.segment_ratios.iter().sum();
+
+        Self {
+            segments,
+
+            config,
+            total_ratio,
+
+            adapter: A::new(),
+        }
+    }
+
+    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
+
+            assert!(!link.as_ref().is_linked());
+
+            let priority = *self.adapter.item2priority(item);
+            link.as_mut().priority = priority.into();
+
+            self.segments[priority.into()].push_back(link);
+
+            self.rebalance();
+        }
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<A::PointerOps as PointerOps>::Pointer,
+    ) -> <A::PointerOps as PointerOps>::Pointer {
+        unsafe {
+            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
+
+            assert!(link.as_ref().is_linked());
+
+            let priority = link.as_ref().priority;
+            self.segments[priority]
+                .iter_mut_from_raw(link.as_ref().link_queue.raw())
+                .remove()
+                .unwrap();
+
+            self.rebalance();
+
+            self.adapter.pointer_ops().from_raw(item)
+        }
+    }
+
+    fn access(&mut self, _ptr: &<A::PointerOps as PointerOps>::Pointer) {}
+
+    fn rebalance(&mut self) {
+        unsafe {
+            let total: usize = self.segments.iter().map(|queue| queue.len()).sum();
+
+            // Rebalance from highest-pri segment to lowest-pri segment. This means the
+            // lowest-pri segment can grow to far larger than its ratio suggests. This
+            // is okay, as we only need higher-pri segments for items that are deemed
+            // important.
+            // e.g. {[a, b, c], [d], [e]} is a valid state for a SFIFO with 3 segments
+            //      and a segment ratio of [1, 1, 1]
+            for high in (1..self.segments.len()).rev() {
+                let low = high - 1;
+                let limit = (total as f64 * self.config.segment_ratios[high] as f64
+                    / self.total_ratio as f64) as usize;
+                while self.segments[high].len() > limit {
+                    let mut link = self.segments[high].pop_front().unwrap();
+                    link.as_mut().priority = low;
+                    self.segments[low].push_back(link);
+                }
+            }
+        }
+    }
+
+    fn iter(&self) -> FifoIter<A> {
+        let mut iter_segments = self.segments.iter().map(|queue| queue.iter()).collect_vec();
+
+        iter_segments.iter_mut().for_each(|iter| iter.front());
+
+        FifoIter {
+            lfu: self,
+            iter_segments,
+            segment: 0,
+
+            ptr: ManuallyDrop::new(None),
+        }
+    }
+}
+
+pub struct FifoIter<'a, A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    lfu: &'a Fifo<A>,
+    iter_segments: Vec<DListIter<'a, FifoLinkAdapter>>,
+    segment: usize,
+
+    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+}
+
+impl<'a, A> FifoIter<'a, A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    unsafe fn update_ptr(&mut self, link: NonNull<FifoLink>) {
+        std::mem::forget(self.ptr.take());
+
+        let item = self.lfu.adapter.link2item(link.as_ptr());
+        let ptr = self.lfu.adapter.pointer_ops().from_raw(item);
+        self.ptr = ManuallyDrop::new(Some(ptr));
+    }
+
+    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+        if self.ptr.is_none() {
+            return None;
+        }
+        let ptr = self.ptr.as_ref().unwrap();
+        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        Some(&*raw)
+    }
+}
+
+impl<'a, A> Iterator for FifoIter<'a, A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        unsafe {
+            let mut link = None;
+            while self.segment < self.iter_segments.len() {
+                match self.iter_segments[self.segment].get().map(|l| l.raw()) {
+                    Some(l) => {
+                        self.iter_segments[self.segment].next();
+                        link = Some(l);
+                        break;
+                    }
+                    None => self.segment += 1,
+                }
+            }
+            match link {
+                None => None,
+                Some(link) => {
+                    self.update_ptr(link);
+                    self.ptr()
+                }
+            }
+        }
+    }
+}
+
+// unsafe impl `Send + Sync` for structs with `NonNull` usage
+
+unsafe impl<A> Send for Fifo<A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+unsafe impl<A> Sync for Fifo<A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+unsafe impl Send for FifoLink {}
+unsafe impl Sync for FifoLink {}
+
+unsafe impl<'a, A> Send for FifoIter<'a, A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+unsafe impl<'a, A> Sync for FifoIter<'a, A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+impl<A> EvictionPolicy<A> for Fifo<A>
+where
+    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Link = FifoLink;
+
+    type Config = FifoConfig;
+
+    type E<'e> = FifoIter<'e, A>;
+
+    fn new(config: Self::Config) -> Self {
+        Self::new(config)
+    }
+
+    fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        tracing::debug!("[lfu] insert {:?}", ptr);
+        self.insert(ptr)
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
+    ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
+        tracing::debug!("[lfu] remove {:?}", ptr);
+        self.remove(ptr)
+    }
+
+    fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        tracing::debug!("[lfu] access {:?}", ptr);
+        self.access(ptr)
+    }
+
+    fn iter(&self) -> Self::E<'_> {
+        self.iter()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use itertools::Itertools;
+
+    use crate::priority_adapter;
+
+    use super::*;
+
+    #[derive(Debug)]
+    struct FifoItem {
+        link: FifoLink,
+        key: u64,
+        priority: usize,
+    }
+
+    impl FifoItem {
+        fn new(key: u64, priority: usize) -> Self {
+            Self {
+                link: FifoLink::default(),
+                key,
+                priority,
+            }
+        }
+    }
+
+    intrusive_adapter! { FifoItemAdapter = Arc<FifoItem>: FifoItem { link: FifoLink } }
+    priority_adapter! { FifoItemAdapter = FifoItem { priority: usize } }
+
+    #[test]
+    fn test_fifo_simple() {
+        let config = FifoConfig {
+            segment_ratios: vec![6, 3, 1],
+        };
+        let mut fifo = Fifo::<FifoItemAdapter>::new(config);
+
+        let mut items = vec![];
+
+        // see comments in `rebalance`
+        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 8], [9]]
+        for key in 0..10 {
+            let item = Arc::new(FifoItem::new(key, 2));
+            fifo.push(item.clone());
+            items.push(item);
+        }
+        let v = fifo.iter().map(|item| item.key).collect_vec();
+        assert_eq!(v, (0..10).collect_vec());
+        let lens = fifo.segments.iter().map(|queue| queue.len()).collect_vec();
+        assert_eq!(lens, vec![7, 2, 1]);
+
+        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 8, 10], [9]]
+        let item = Arc::new(FifoItem::new(10, 1));
+        fifo.push(item.clone());
+        items.push(item);
+        let v = fifo.iter().map(|item| item.key).collect_vec();
+        assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 9]);
+
+        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 10], [9]]
+        fifo.remove(&items[8]);
+        let v = fifo.iter().map(|item| item.key).collect_vec();
+        assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 10, 9]);
+
+        drop(fifo);
+
+        for item in items {
+            assert_eq!(Arc::strong_count(&item), 1);
+        }
+    }
+}
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 9d5870fb..c612b280 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -126,7 +126,7 @@ intrusive_adapter! { LfuLinkMainDListAdapter = NonNull<LfuLink>: LfuLink { link_
 /// This default to 1%. There's no need to tune this parameter.
 pub struct Lfu<A>
 where
-    A: KeyAdapter<Link = LfuLink>,
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     /// tiny lru list
@@ -156,7 +156,7 @@ where
 
 impl<A> Drop for Lfu<A>
 where
-    A: KeyAdapter<Link = LfuLink>,
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     fn drop(&mut self) {
@@ -172,7 +172,7 @@ where
 
 impl<A> Lfu<A>
 where
-    A: KeyAdapter<Link = LfuLink>,
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     pub fn new(config: LfuConfig) -> Self {
@@ -401,8 +401,7 @@ where
 
 pub struct LfuIter<'a, A>
 where
-    A: KeyAdapter<Link = LfuLink>,
-
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     lfu: &'a Lfu<A>,
@@ -414,7 +413,7 @@ where
 
 impl<'a, A> LfuIter<'a, A>
 where
-    A: KeyAdapter<Link = LfuLink>,
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     unsafe fn update_ptr(&mut self, link: NonNull<LfuLink>) {
@@ -437,8 +436,7 @@ where
 
 impl<'a, A> Iterator for LfuIter<'a, A>
 where
-    A: KeyAdapter<Link = LfuLink>,
-
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     type Item = &'a <A::PointerOps as PointerOps>::Pointer;
@@ -485,15 +483,13 @@ where
 
 unsafe impl<A> Send for Lfu<A>
 where
-    A: KeyAdapter<Link = LfuLink>,
-
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 unsafe impl<A> Sync for Lfu<A>
 where
-    A: KeyAdapter<Link = LfuLink>,
-
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
@@ -503,23 +499,21 @@ unsafe impl Sync for LfuLink {}
 
 unsafe impl<'a, A> Send for LfuIter<'a, A>
 where
-    A: KeyAdapter<Link = LfuLink>,
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
 
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 unsafe impl<'a, A> Sync for LfuIter<'a, A>
 where
-    A: KeyAdapter<Link = LfuLink>,
-
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 
 impl<A> EvictionPolicy<A> for Lfu<A>
 where
-    A: KeyAdapter<Link = LfuLink>,
-
+    A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     type Link = LfuLink;
@@ -624,10 +618,7 @@ mod tests {
             lfu.iter().map(|item| item.key).collect_vec()
         );
 
-        let to_remove_items = lfu.iter().cloned().collect_vec();
-        for item in to_remove_items {
-            lfu.remove(&item);
-        }
+        drop(lfu);
 
         for item in items {
             assert_eq!(Arc::strong_count(&item), 1);
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index db8f9ac1..921b67cd 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -31,13 +31,13 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
-        adapter::{KeyAdapter, Link},
+        adapter::{Adapter, Link},
         pointer::PointerOps,
     },
     intrusive_adapter,
 };
 
-use super::{Adapter, EvictionPolicy};
+use super::EvictionPolicy;
 
 #[derive(Clone, Debug)]
 pub struct LruConfig {
@@ -68,7 +68,7 @@ intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DLis
 
 pub struct Lru<A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     /// lru list
@@ -87,7 +87,7 @@ where
 
 impl<A> Drop for Lru<A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     fn drop(&mut self) {
@@ -103,7 +103,7 @@ where
 
 impl<A> Lru<A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     fn new(config: LruConfig) -> Self {
@@ -277,7 +277,7 @@ where
 
 pub struct LruIter<'a, A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     lru: &'a Lru<A>,
@@ -288,7 +288,7 @@ where
 
 impl<'a, A> LruIter<'a, A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     unsafe fn update_ptr(&mut self, link: NonNull<LruLink>) {
@@ -311,7 +311,7 @@ where
 
 impl<'a, A> Iterator for LruIter<'a, A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     type Item = &'a <A::PointerOps as PointerOps>::Pointer;
@@ -336,13 +336,13 @@ where
 
 unsafe impl<A> Send for Lru<A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 unsafe impl<A> Sync for Lru<A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
@@ -352,20 +352,20 @@ unsafe impl Sync for LruLink {}
 
 unsafe impl<'a, A> Send for LruIter<'a, A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 unsafe impl<'a, A> Sync for LruIter<'a, A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 
 impl<A> EvictionPolicy<A> for Lru<A>
 where
-    A: KeyAdapter<Link = LruLink>,
+    A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     type Link = LruLink;
@@ -456,8 +456,7 @@ mod tests {
 
         assert_eq!(vec![0, 1], lru.iter().map(|item| item.key).collect_vec());
 
-        lru.remove(&handles[0]);
-        lru.remove(&handles[1]);
+        drop(lru);
 
         for handle in handles {
             assert_eq!(Arc::strong_count(&handle), 1);
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index df04b217..f3fce481 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -13,7 +13,7 @@
 //  limitations under the License.
 
 use crate::core::{
-    adapter::{Adapter, KeyAdapter, Link},
+    adapter::{Adapter, Link},
     pointer::PointerOps,
 };
 
@@ -23,7 +23,7 @@ pub trait Config = Send + Sync + 'static + Debug;
 
 pub trait EvictionPolicy<A>: Send + Sync + 'static
 where
-    A: KeyAdapter<Link = Self::Link>,
+    A: Adapter<Link = Self::Link>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     type Link: Link;
@@ -61,5 +61,6 @@ where
     }
 }
 
+pub mod fifo;
 pub mod lfu;
 pub mod lru;
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index ee310788..b007d557 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -57,3 +57,16 @@ pub type LfuFsStore<K, V, AP, RP> = store::Store<
     RP,
     foyer_intrusive::eviction::lfu::LfuLink,
 >;
+
+pub type FifoFsStore<K, V, AP, RP> = store::Store<
+    K,
+    V,
+    AlignedAllocator,
+    device::fs::FsDevice,
+    foyer_intrusive::eviction::fifo::Fifo<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
+    >,
+    AP,
+    RP,
+    foyer_intrusive::eviction::fifo::FifoLink,
+>;

From 98a0af70f46b6979ed277e4a88a07da271b410fc Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 3 Jul 2023 16:03:33 +0800
Subject: [PATCH 027/261] feat: export mods (#37)

* feat: export mods

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* make cargo sort happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.lock       | 2 ++
 foyer/Cargo.toml | 2 ++
 foyer/src/lib.rs | 4 ++++
 3 files changed, 8 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 0d725ec4..ac3c9ae5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -344,6 +344,8 @@ dependencies = [
  "cmsketch",
  "crossbeam",
  "foyer-common",
+ "foyer-intrusive",
+ "foyer-storage",
  "futures",
  "hdrhistogram",
  "itertools",
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index c9bdf5b4..07a22aea 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -13,6 +13,8 @@ bytes = "1"
 cmsketch = "0.1"
 crossbeam = "0.8"
 foyer-common = { path = "../foyer-common" }
+foyer-intrusive = { path = "../foyer-intrusive" }
+foyer-storage = { path = "../foyer-storage" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 213a7b3f..494f718a 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -26,3 +26,7 @@ impl Weight for Vec<u8> {
         self.len()
     }
 }
+
+pub use foyer_common as common;
+pub use foyer_intrusive as intrusive;
+pub use foyer_storage as storage;

From a8a56ccc19756d4f4b7ac253971163b4dcfda637 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 3 Jul 2023 16:13:35 +0800
Subject: [PATCH 028/261] fix: export extern crate (#38)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 494f718a..ba5af241 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -27,6 +27,6 @@ impl Weight for Vec<u8> {
     }
 }
 
-pub use foyer_common as common;
-pub use foyer_intrusive as intrusive;
-pub use foyer_storage as storage;
+pub extern crate foyer_common as common;
+pub extern crate foyer_intrusive as intrusive;
+pub extern crate foyer_storage as storage;

From 09bf57a161b4e4ac3cd5b5e7a39853cf5e1d326c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 3 Jul 2023 16:18:52 +0800
Subject: [PATCH 029/261] Revert "fix: export extern crate (#38)" (#39)

This reverts commit a8a56ccc19756d4f4b7ac253971163b4dcfda637.
---
 foyer/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index ba5af241..494f718a 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -27,6 +27,6 @@ impl Weight for Vec<u8> {
     }
 }
 
-pub extern crate foyer_common as common;
-pub extern crate foyer_intrusive as intrusive;
-pub extern crate foyer_storage as storage;
+pub use foyer_common as common;
+pub use foyer_intrusive as intrusive;
+pub use foyer_storage as storage;

From 691054a57ee46ea142b58e7022e4509283211e77 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 3 Jul 2023 16:37:25 +0800
Subject: [PATCH 030/261] fix: disable O_DIRECT on non-linux targets (#40)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/device/fs.rs | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 837bc927..f250510a 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -14,10 +14,7 @@
 
 use std::{
     fs::{create_dir_all, File, OpenOptions},
-    os::{
-        fd::{AsRawFd, RawFd},
-        unix::prelude::OpenOptionsExt,
-    },
+    os::fd::{AsRawFd, RawFd},
     path::PathBuf,
     sync::Arc,
 };
@@ -64,6 +61,7 @@ impl FsDeviceConfig {
 struct FsDeviceInner {
     config: FsDeviceConfig,
 
+    #[allow(unused)]
     dir: File,
 
     files: Vec<File>,
@@ -125,12 +123,12 @@ impl Device for FsDevice {
         Ok(res)
     }
 
+    #[cfg(target_os = "linux")]
     async fn flush(&self) -> Result<()> {
         let fd = self.inner.dir.as_raw_fd();
         // Commit fs cache to disk. Linux waits for I/O completions.
         //
         // See also [syncfs(2)](https://man7.org/linux/man-pages/man2/sync.2.html)
-        #[cfg(target_os = "linux")]
         asyncify(move || {
             nix::unistd::syncfs(fd)?;
             Ok(())
@@ -142,6 +140,13 @@ impl Device for FsDevice {
         Ok(())
     }
 
+    #[cfg(not(target_os = "linux"))]
+    async fn flush(&self) -> Result<()> {
+        // TODO(MrCroxx): track dirty files and call fsync(2) on them on other target os.
+
+        Ok(())
+    }
+
     fn capacity(&self) -> usize {
         self.inner.config.capacity
     }
@@ -190,10 +195,14 @@ impl FsDevice {
             .map(|i| {
                 let path = config.dir.clone().join(Self::filename(i as RegionId));
                 async move {
+                    #[cfg(target_os = "linux")]
+                    use std::os::unix::prelude::OpenOptionsExt;
+
                     let mut opts = OpenOptions::new();
                     opts.create(true);
                     opts.write(true);
                     opts.read(true);
+                    #[cfg(target_os = "linux")]
                     opts.custom_flags(libc::O_DIRECT);
 
                     let file = opts.open(path).map_err(Error::io)?;

From f067d1cc2fc3657788a51559bffbd3b371f46317 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 4 Jul 2023 15:12:00 +0800
Subject: [PATCH 031/261] fix: deadlock (#43)

* fix: deadlock

fix deadlocks:

1. exclusive lock was not `Send` but unsafe impled.
2. submit flush task await blocks `set_region_evictable`
3. no new reclamation task created after `set_region_evctable` if all
   write process are waiting

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml       |  37 +-
 .github/workflows/main.yml          |  36 +-
 .github/workflows/pull-request.yml  |  36 +-
 Cargo.lock                          | 827 ++++++++++++++++++++++------
 foyer-storage-bench/Cargo.toml      |   5 +
 foyer-storage-bench/src/main.rs     |  42 +-
 foyer-storage/Cargo.toml            |   6 +-
 foyer-storage/src/device/fs.rs      |   2 +
 foyer-storage/src/device/mod.rs     |   1 +
 foyer-storage/src/flusher.rs        |  45 +-
 foyer-storage/src/reclaimer.rs      |  15 +-
 foyer-storage/src/region.rs         | 265 ++++-----
 foyer-storage/src/region_manager.rs |  77 +--
 foyer-storage/src/store.rs          |  11 +-
 14 files changed, 1019 insertions(+), 386 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index fff377d7..88b1be1c 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -70,6 +70,37 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  deadlock:
+    name: run with single worker thread and deadlock detection
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          # components: rustfmt, clippy, llvm-tools-preview
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-storage-bench with single worker thread and deadlock detection
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '--cfg tokio_unstable'
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo build --all --features deadlock &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -95,8 +126,8 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+          RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
\ No newline at end of file
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f00667d2..ce135f8c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -77,6 +77,36 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  deadlock:
+    name: run with single worker thread and deadlock detection
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-storage-bench with single worker thread and deadlock detection
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '--cfg tokio_unstable'
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo build --all --features deadlock &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -101,11 +131,11 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+          RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
 
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index bfb037f0..b4beb2af 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -76,6 +76,36 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  deadlock:
+    name: run with single worker thread and deadlock detection
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-storage-bench with single worker thread and deadlock detection
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '--cfg tokio_unstable'
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo build --all --features deadlock &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -100,11 +130,11 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+          RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
 concurrency:
   group: environment-${{ github.ref }}
   cancel-in-progress: true
diff --git a/Cargo.lock b/Cargo.lock
index ac3c9ae5..86daa791 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "addr2line"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler"
 version = "1.0.2"
@@ -25,15 +34,15 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d"
+checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee"
+checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
 dependencies = [
  "utf8parse",
 ]
@@ -44,7 +53,7 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -54,9 +63,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
 dependencies = [
  "anstyle",
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.71"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
+
 [[package]]
 name = "async-channel"
 version = "1.8.0"
@@ -70,13 +85,13 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.68"
+version = "0.1.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
+checksum = "7b2d0f03b3640e3a630367e40c468cb7f309529c708ed1d88597047b0e7c6ef7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -85,12 +100,78 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "axum"
+version = "0.6.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39"
+dependencies = [
+ "async-trait",
+ "axum-core",
+ "bitflags 1.3.2",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "sync_wrapper",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "mime",
+ "rustversion",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "backtrace"
+version = "0.3.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
+[[package]]
+name = "base64"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -135,9 +216,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "4.3.0"
+version = "4.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc"
+checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -146,27 +227,26 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.3.0"
+version = "4.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990"
+checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d"
 dependencies = [
  "anstream",
  "anstyle",
- "bitflags 1.3.2",
  "clap_lex",
  "strsim",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.3.0"
+version = "4.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b"
+checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -199,6 +279,42 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "console-api"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e"
+dependencies = [
+ "prost",
+ "prost-types",
+ "tonic",
+ "tracing-core",
+]
+
+[[package]]
+name = "console-subscriber"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57ab2224a0311582eb03adba4caaf18644f7b1f10a760803a803b9b605187fc7"
+dependencies = [
+ "console-api",
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "futures",
+ "hdrhistogram",
+ "humantime",
+ "prost-types",
+ "serde",
+ "serde_json",
+ "thread_local",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tracing",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.3.2"
@@ -245,14 +361,14 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.14"
+version = "0.9.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
+checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
 dependencies = [
  "autocfg",
  "cfg-if",
  "crossbeam-utils",
- "memoffset 0.8.0",
+ "memoffset 0.9.0",
  "scopeguard",
 ]
 
@@ -268,9 +384,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.15"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
 dependencies = [
  "cfg-if",
 ]
@@ -289,7 +405,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
 dependencies = [
  "errno-dragonfly",
  "libc",
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -317,6 +433,12 @@ dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
+
 [[package]]
 name = "flate2"
 version = "1.0.26"
@@ -410,6 +532,7 @@ dependencies = [
  "nix",
  "parking_lot",
  "paste",
+ "pin-project",
  "prometheus",
  "rand",
  "rand_mt",
@@ -426,6 +549,7 @@ version = "0.1.0"
 dependencies = [
  "bytesize",
  "clap",
+ "console-subscriber",
  "foyer-intrusive",
  "foyer-storage",
  "futures",
@@ -498,7 +622,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -533,22 +657,53 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
 dependencies = [
  "cfg-if",
  "libc",
  "wasi",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+
+[[package]]
+name = "h2"
+version = "0.3.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049"
+dependencies = [
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "hdrhistogram"
 version = "7.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8"
 dependencies = [
- "base64",
+ "base64 0.13.1",
  "byteorder",
  "crossbeam-channel",
  "flate2",
@@ -564,18 +719,95 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
 [[package]]
 name = "hermit-abi"
-version = "0.2.6"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+
+[[package]]
+name = "http"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
 dependencies = [
- "libc",
+ "bytes",
+ "fnv",
+ "itoa",
 ]
 
 [[package]]
-name = "hermit-abi"
-version = "0.3.1"
+name = "http-body"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
+
+[[package]]
+name = "httpdate"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
+[[package]]
+name = "hyper"
+version = "0.14.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
 
 [[package]]
 name = "instant"
@@ -588,25 +820,24 @@ dependencies = [
 
 [[package]]
 name = "io-lifetimes"
-version = "1.0.10"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
+checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
 dependencies = [
- "hermit-abi 0.3.1",
+ "hermit-abi",
  "libc",
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
 [[package]]
 name = "is-terminal"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
+checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb"
 dependencies = [
- "hermit-abi 0.3.1",
- "io-lifetimes",
- "rustix",
- "windows-sys 0.48.0",
+ "hermit-abi",
+ "rustix 0.38.2",
+ "windows-sys",
 ]
 
 [[package]]
@@ -618,6 +849,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itoa"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0aa48fab2893d8a49caa94082ae8488f4e1050d73b367881dcd2198f4199fd8"
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -626,9 +863,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.144"
+version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
 
 [[package]]
 name = "linux-raw-sys"
@@ -636,11 +873,17 @@ version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
+
 [[package]]
 name = "lock_api"
-version = "0.4.9"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
+checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -648,12 +891,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.17"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
 
 [[package]]
 name = "matchers"
@@ -664,6 +904,12 @@ dependencies = [
  "regex-automata",
 ]
 
+[[package]]
+name = "matchit"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -688,6 +934,21 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "mime"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -705,14 +966,13 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.6"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
+checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
 dependencies = [
  "libc",
- "log",
  "wasi",
- "windows-sys 0.45.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -760,19 +1020,28 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.2.6",
+ "hermit-abi",
  "libc",
 ]
 
+[[package]]
+name = "object"
+version = "0.31.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
-version = "1.17.1"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "overload"
@@ -792,15 +1061,18 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.7"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
+checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
 dependencies = [
+ "backtrace",
  "cfg-if",
  "libc",
- "redox_syscall 0.2.16",
+ "petgraph",
+ "redox_syscall 0.3.5",
  "smallvec",
- "windows-sys 0.45.0",
+ "thread-id",
+ "windows-targets",
 ]
 
 [[package]]
@@ -809,11 +1081,47 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
 
+[[package]]
+name = "percent-encoding"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
+
+[[package]]
+name = "petgraph"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
+[[package]]
+name = "pin-project"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.23",
+]
+
 [[package]]
 name = "pin-project-lite"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
 
 [[package]]
 name = "pin-utils"
@@ -829,9 +1137,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.56"
+version = "1.0.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
+checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
 dependencies = [
  "unicode-ident",
 ]
@@ -851,6 +1159,38 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "prost"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
+dependencies = [
+ "prost",
+]
+
 [[package]]
 name = "protobuf"
 version = "2.28.0"
@@ -859,9 +1199,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
 
 [[package]]
 name = "quote"
-version = "1.0.27"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500"
+checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
 dependencies = [
  "proc-macro2",
 ]
@@ -898,9 +1238,9 @@ dependencies = [
 
 [[package]]
 name = "rand_mt"
-version = "4.2.1"
+version = "4.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77717f0c31ae3827865b59615cebc0e54374bd802bfe0ef067684a50bcfb10a9"
+checksum = "49e018c6ded60e5252609887c12eb3ca2592e9248c5894a7db3975c8a7a1e2df"
 dependencies = [
  "rand_core",
 ]
@@ -953,26 +1293,88 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+
 [[package]]
 name = "rustix"
-version = "0.37.19"
+version = "0.37.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
+checksum = "8818fa822adcc98b18fedbb3632a6a33213c070556b5aa7c4c8cc21cff565c4c"
 dependencies = [
  "bitflags 1.3.2",
  "errno",
  "io-lifetimes",
  "libc",
- "linux-raw-sys",
- "windows-sys 0.48.0",
+ "linux-raw-sys 0.3.8",
+ "windows-sys",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.38.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4"
+dependencies = [
+ "bitflags 2.3.3",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.4.3",
+ "windows-sys",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"
+
+[[package]]
+name = "ryu"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "serde"
+version = "1.0.165"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c939f902bb7d0ccc5bce4f03297e161543c2dcb30914faf032c2bd0b7a0d48fc"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.165"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6eaae920e25fffe4019b75ff65e7660e72091e59dd204cb5849bbd6a3fd343d7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.23",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "sharded-slab"
 version = "0.1.4"
@@ -1006,6 +1408,16 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
+[[package]]
+name = "socket2"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -1020,26 +1432,44 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "2.0.15"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822"
+checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
 dependencies = [
  "proc-macro2",
  "quote",
  "unicode-ident",
 ]
 
+[[package]]
+name = "sync_wrapper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+
 [[package]]
 name = "tempfile"
-version = "3.5.0"
+version = "3.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998"
+checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
 dependencies = [
+ "autocfg",
  "cfg-if",
  "fastrand",
  "redox_syscall 0.3.5",
- "rustix",
- "windows-sys 0.45.0",
+ "rustix 0.37.22",
+ "windows-sys",
 ]
 
 [[package]]
@@ -1059,7 +1489,18 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
+]
+
+[[package]]
+name = "thread-id"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ee93aa2b8331c0fec9091548843f2c90019571814057da3b783f9de09349d73"
+dependencies = [
+ "libc",
+ "redox_syscall 0.2.16",
+ "winapi",
 ]
 
 [[package]]
@@ -1074,18 +1515,32 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.28.1"
+version = "1.29.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
+checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
 dependencies = [
  "autocfg",
+ "backtrace",
+ "bytes",
  "libc",
  "mio",
  "num_cpus",
  "pin-project-lite",
  "signal-hook-registry",
+ "socket2",
  "tokio-macros",
- "windows-sys 0.48.0",
+ "tracing",
+ "windows-sys",
+]
+
+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
 ]
 
 [[package]]
@@ -1096,9 +1551,94 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
+]
+
+[[package]]
+name = "tokio-stream"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "tonic"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
+dependencies = [
+ "async-trait",
+ "axum",
+ "base64 0.21.2",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
 ]
 
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "indexmap",
+ "pin-project",
+ "pin-project-lite",
+ "rand",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+
+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
 [[package]]
 name = "tracing"
 version = "0.1.37"
@@ -1113,13 +1653,13 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.24"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74"
+checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -1161,6 +1701,12 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "try-lock"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
+
 [[package]]
 name = "twox-hash"
 version = "1.6.3"
@@ -1174,9 +1720,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0"
 
 [[package]]
 name = "utf8parse"
@@ -1190,6 +1736,15 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -1218,132 +1773,66 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "windows-sys"
-version = "0.45.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
-dependencies = [
- "windows-targets 0.42.2",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets 0.48.0",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
-dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
+ "windows-targets",
 ]
 
 [[package]]
 name = "windows-targets"
-version = "0.48.0"
+version = "0.48.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
 dependencies = [
- "windows_aarch64_gnullvm 0.48.0",
- "windows_aarch64_msvc 0.48.0",
- "windows_i686_gnu 0.48.0",
- "windows_i686_msvc 0.48.0",
- "windows_x86_64_gnu 0.48.0",
- "windows_x86_64_gnullvm 0.48.0",
- "windows_x86_64_msvc 0.48.0",
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
 ]
 
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
-
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
 
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
-
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
 
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.0"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 3254b026..9145a090 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -10,6 +10,7 @@ license = "Apache-2.0"
 [dependencies]
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
+console-subscriber = { version = "0.1", optional = true }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-storage = { path = "../foyer-storage" }
 futures = "0.3"
@@ -31,3 +32,7 @@ tokio = { version = "1", features = [
 ] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+[features]
+deadlock = ["parking_lot/deadlock_detection", "foyer-storage/deadlock"]
+tokio-console = ["console-subscriber"]
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 5e5b21c0..44c13a69 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -42,7 +42,6 @@ use rand::{rngs::StdRng, Rng, SeedableRng};
 
 use rate::RateLimiter;
 use tokio::sync::oneshot;
-use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt};
 use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
 
 #[derive(Parser, Debug, Clone)]
@@ -130,10 +129,43 @@ fn is_send_sync_static<T: Send + Sync + 'static>() {}
 async fn main() {
     is_send_sync_static::<TStore>();
 
-    tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer())
-        .with(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
+    #[cfg(feature = "tokio-console")]
+    console_subscriber::init();
+
+    #[cfg(not(feature = "tokio-console"))]
+    {
+        use tracing_subscriber::{prelude::*, EnvFilter};
+
+        tracing_subscriber::registry()
+            .with(
+                tracing_subscriber::fmt::layer()
+                    // .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
+                    .with_line_number(true),
+            )
+            .with(EnvFilter::from_default_env())
+            .init();
+    }
+
+    #[cfg(feature = "deadlock")]
+    {
+        std::thread::spawn(move || loop {
+            std::thread::sleep(Duration::from_secs(1));
+            let deadlocks = parking_lot::deadlock::check_deadlock();
+            if deadlocks.is_empty() {
+                continue;
+            }
+
+            println!("{} deadlocks detected", deadlocks.len());
+            for (i, threads) in deadlocks.iter().enumerate() {
+                println!("Deadlock #{}", i);
+                for t in threads {
+                    println!("Thread Id {:#?}", t.thread_id());
+                    println!("{:#?}", t.backtrace());
+                }
+            }
+            panic!()
+        });
+    }
 
     let args = Args::parse();
     args.verify();
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index f289b25b..d3006846 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -20,8 +20,9 @@ itertools = "0.10.5"
 libc = "0.2"
 memoffset = "0.8"
 nix = { version = "0.26", features = ["fs", "mman"] }
-parking_lot = { version = "0.12", features = ["arc_lock"] }
+parking_lot = "0.12"
 paste = "1.0"
+pin-project = "1"
 prometheus = "0.13"
 rand = "0.8.5"
 thiserror = "1"
@@ -42,3 +43,6 @@ clap = { version = "4", features = ["derive"] }
 hdrhistogram = "7"
 rand_mt = "4.2.1"
 tempfile = "3"
+
+[features]
+deadlock = ["parking_lot/deadlock_detection"]
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index f250510a..85808564 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -83,6 +83,7 @@ impl Device for FsDevice {
         Self::open(config).await
     }
 
+    #[tracing::instrument(level = "trace", skip(self, buf))]
     async fn write(
         &self,
         buf: impl IoBuf,
@@ -103,6 +104,7 @@ impl Device for FsDevice {
         Ok(res)
     }
 
+    #[tracing::instrument(level = "trace", skip(self, buf))]
     async fn read(
         &self,
         mut buf: impl IoBufMut,
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 6436e69d..2197b912 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -70,6 +70,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     }
 }
 
+#[tracing::instrument(level = "trace", skip(f))]
 async fn asyncify<F, T>(f: F) -> error::Result<T>
 where
     F: FnOnce() -> error::Result<T> + Send + 'static,
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 623ae792..f8359145 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -18,7 +18,7 @@ use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use itertools::Itertools;
 use tokio::sync::{
-    mpsc::{channel, error::TrySendError, Receiver, Sender},
+    mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
     Mutex,
 };
 
@@ -38,7 +38,7 @@ pub struct FlushTask {
 struct FlusherInner {
     sequence: usize,
 
-    task_txs: Vec<Sender<FlushTask>>,
+    task_txs: Vec<UnboundedSender<FlushTask>>,
 }
 
 pub struct Flusher {
@@ -72,8 +72,10 @@ impl Flusher {
         let mut inner = self.inner.lock().await;
 
         #[allow(clippy::type_complexity)]
-        let (mut txs, rxs): (Vec<Sender<FlushTask>>, Vec<Receiver<FlushTask>>) =
-            (0..self.runners).map(|_| channel(1)).unzip();
+        let (mut txs, rxs): (
+            Vec<UnboundedSender<FlushTask>>,
+            Vec<UnboundedReceiver<FlushTask>>,
+        ) = (0..self.runners).map(|_| unbounded_channel()).unzip();
         inner.task_txs.append(&mut txs);
 
         let runners = rxs
@@ -100,23 +102,7 @@ impl Flusher {
         let mut inner = self.inner.lock().await;
         let submittee = inner.sequence % inner.task_txs.len();
         inner.sequence += 1;
-        inner.task_txs[submittee]
-            .send(task)
-            .await
-            .map_err(Error::other)
-    }
-
-    pub async fn try_submit(&self, task: FlushTask) -> Result<()> {
-        let mut inner = self.inner.lock().await;
-        let submittee = inner.sequence % inner.task_txs.len();
-        match inner.task_txs[submittee].try_send(task) {
-            Ok(()) => {
-                inner.sequence += 1;
-                Ok(())
-            }
-            Err(TrySendError::Full(_)) => Err(Error::ChannelFull),
-            Err(e) => Err(Error::Other(e.to_string())),
-        }
+        inner.task_txs[submittee].send(task).map_err(Error::other)
     }
 }
 
@@ -127,7 +113,7 @@ where
     E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
     EL: Link,
 {
-    task_rx: Receiver<FlushTask>,
+    task_rx: UnboundedReceiver<FlushTask>,
     buffers: Arc<AsyncQueue<Vec<u8, A>>>,
 
     region_manager: Arc<RegionManager<A, D, E, EL>>,
@@ -149,6 +135,8 @@ where
 
                 let region = self.region_manager.region(&task.region_id);
 
+                tracing::trace!("[flusher] step 1");
+
                 {
                     // step 1: write buffer back to device
                     let slice = region.load(.., 0).await?.unwrap();
@@ -156,7 +144,7 @@ where
                     // wait all physical readers (from previous version) and writers done
                     let guard = region.exclusive(false, true, false).await;
 
-                    tracing::info!("[flusher] write region {} back to device", task.region_id);
+                    tracing::trace!("[flusher] write region {} back to device", task.region_id);
 
                     let mut offset = 0;
                     let len = region.device().io_size();
@@ -164,8 +152,6 @@ where
                         let start = offset;
                         let end = std::cmp::min(offset + len, region.device().region_size());
 
-                        tracing::trace!("write region {} [{}..{}]", region.id(), start, end);
-
                         let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
                         region
                             .device()
@@ -173,12 +159,12 @@ where
                             .await?;
                         offset += len;
                     }
-
                     drop(guard);
-
-                    tracing::debug!("[flusher] drop exclusive guard");
+                    slice.destroy().await;
                 }
 
+                tracing::trace!("[flusher] step 2");
+
                 let buffer = {
                     // step 2: detach buffer
                     let mut guard = region.exclusive(false, false, true).await;
@@ -197,9 +183,10 @@ where
                     buffer
                 };
 
+                tracing::trace!("[flusher] step 3");
+
                 // step 3: release buffer
                 self.buffers.release(buffer);
-
                 self.region_manager.set_region_evictable(&region.id()).await;
 
                 tracing::info!("[flusher] finish flush task, region: {}", task.region_id);
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index d921cfe9..7e5da64c 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -32,7 +32,7 @@ use foyer_common::{
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use itertools::Itertools;
 use tokio::sync::{
-    mpsc::{channel, error::TrySendError, Receiver, Sender},
+    mpsc::{channel, Receiver, Sender},
     Mutex,
 };
 
@@ -122,19 +122,6 @@ impl Reclaimer {
             .await
             .map_err(Error::other)
     }
-
-    pub async fn try_submit(&self, task: ReclaimTask) -> Result<()> {
-        let mut inner = self.inner.lock().await;
-        let submittee = inner.sequence % inner.task_txs.len();
-        match inner.task_txs[submittee].try_send(task) {
-            Ok(()) => {
-                inner.sequence += 1;
-                Ok(())
-            }
-            Err(TrySendError::Full(_)) => Err(Error::ChannelFull),
-            Err(e) => Err(Error::Other(e.to_string())),
-        }
-    }
 }
 
 struct Runner<K, V, A, D, EP, AP, RP, EL>
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 154f3647..3615661d 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -12,16 +12,11 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    ops::{Deref, DerefMut, RangeBounds},
-    pin::Pin,
-    sync::Arc,
-    task::{Context, Poll, Waker},
-};
+use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, pin::Pin, sync::Arc, task::Waker};
 
-use futures::Future;
-use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock};
+use futures::future::BoxFuture;
+use tokio::sync::{OwnedRwLockWriteGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
+use tracing::instrument;
 
 use crate::{
     device::{BufferAllocator, Device},
@@ -62,7 +57,7 @@ where
     buffered_readers: usize,
     physical_readers: usize,
 
-    wakers: Vec<Waker>,
+    wakers: HashMap<usize, Waker>,
 }
 
 #[derive(Debug, Clone)]
@@ -73,7 +68,7 @@ where
 {
     id: RegionId,
 
-    inner: Arc<RwLock<RegionInner<A>>>,
+    inner: ErwLock<A>,
 
     device: D,
 }
@@ -107,26 +102,26 @@ where
             buffered_readers: 0,
             physical_readers: 0,
 
-            wakers: vec![],
+            wakers: HashMap::default(),
         };
         Self {
             id,
-            inner: Arc::new(RwLock::new(inner)),
+            inner: ErwLock::new(inner),
             device,
         }
     }
 
-    pub fn allocate(&self, size: usize) -> AllocateResult {
-        let callback = {
+    pub async fn allocate(&self, size: usize) -> AllocateResult {
+        let future = {
             let inner = self.inner.clone();
-            move || {
-                let mut guard = inner.write();
+            async move {
+                let mut guard = inner.write().await;
                 guard.writers -= 1;
                 guard.wake_all();
             }
         };
 
-        let mut inner = self.inner.write();
+        let mut inner = self.inner.write().await;
 
         inner.writers += 1;
         let version = inner.version;
@@ -148,7 +143,7 @@ where
                 region_id,
                 version,
                 offset,
-                callback: Box::new(callback),
+                future: Some(Box::pin(future)),
             };
             AllocateResult::Full { slice, remain }
         } else {
@@ -162,12 +157,13 @@ where
                 region_id,
                 version,
                 offset,
-                callback: Box::new(callback),
+                future: Some(Box::pin(future)),
             };
             AllocateResult::Ok(slice)
         }
     }
 
+    #[tracing::instrument(skip(self, range), fields(start, end))]
     pub async fn load(
         &self,
         range: impl RangeBounds<usize>,
@@ -186,7 +182,8 @@ where
 
         // restrict guard lifetime
         {
-            let mut inner = self.inner.write();
+            let mut inner = self.inner.write().await;
+
             if version != 0 && version != inner.version {
                 return Ok(None);
             }
@@ -197,10 +194,10 @@ where
                 inner.buffered_readers += 1;
                 let allocator = inner.buffer.as_ref().unwrap().allocator().clone();
                 let slice = unsafe { Slice::new(&inner.buffer.as_ref().unwrap()[start..end]) };
-                let callback = {
+                let future = {
                     let inner = self.inner.clone();
-                    move || {
-                        let mut guard = inner.write();
+                    async move {
+                        let mut guard = inner.write().await;
                         guard.buffered_readers -= 1;
                         guard.wake_all();
                     }
@@ -208,7 +205,7 @@ where
                 return Ok(Some(ReadSlice::Slice {
                     slice,
                     allocator: Some(allocator),
-                    callback: Box::new(callback),
+                    future: Some(Box::pin(future)),
                 }));
             }
 
@@ -236,28 +233,30 @@ where
                 .await?
                 != len
             {
-                self.inner.write().physical_readers -= 1;
+                let mut inner = self.inner.write().await;
+                inner.physical_readers -= 1;
+                inner.wake_all();
                 return Ok(None);
             }
             offset += len;
         }
 
-        let callback = {
+        let future = {
             let inner = self.inner.clone();
-            move || {
-                let mut guard = inner.write();
+            async move {
+                let mut guard = inner.write().await;
                 guard.physical_readers -= 1;
                 guard.wake_all();
             }
         };
         Ok(Some(ReadSlice::Owned {
             buf: Some(buf),
-            callback: Box::new(callback),
+            future: Some(Box::pin(future)),
         }))
     }
 
-    pub fn attach_buffer(&self, buf: Vec<u8, A>) {
-        let mut inner = self.inner.write();
+    pub async fn attach_buffer(&self, buf: Vec<u8, A>) {
+        let mut inner = self.inner.write().await;
 
         assert_eq!(inner.writers, 0);
         assert_eq!(inner.buffered_readers, 0);
@@ -265,31 +264,27 @@ where
         inner.attach_buffer(buf);
     }
 
-    pub fn detach_buffer(&self) -> Vec<u8, A> {
-        let mut inner = self.inner.write();
+    pub async fn detach_buffer(&self) -> Vec<u8, A> {
+        let mut inner = self.inner.write().await;
 
         inner.detach_buffer()
     }
 
-    pub fn has_buffer(&self) -> bool {
-        let inner = self.inner.read();
+    pub async fn has_buffer(&self) -> bool {
+        let inner = self.inner.read().await;
         inner.has_buffer()
     }
 
+    #[instrument(skip(self))]
     pub async fn exclusive(
         &self,
         can_write: bool,
         can_buffered_read: bool,
         can_physical_read: bool,
-    ) -> ExclusiveGuard<A> {
-        ExclusiveFuture {
-            inner: self.inner.clone(),
-            can_write,
-            can_buffered_read,
-            can_physical_read,
-            is_waker_set: false,
-        }
-        .await
+    ) -> OwnedRwLockWriteGuard<RegionInner<A>> {
+        self.inner
+            .exclusive(can_write, can_buffered_read, can_physical_read)
+            .await
     }
 
     pub fn id(&self) -> RegionId {
@@ -300,12 +295,12 @@ where
         &self.device
     }
 
-    pub fn version(&self) -> Version {
-        self.inner.read().version
+    pub async fn version(&self) -> Version {
+        self.inner.read().await.version
     }
 
-    pub fn advance(&self) -> Version {
-        let mut inner = self.inner.write();
+    pub async fn advance(&self) -> Version {
+        let mut inner = self.inner.write().await;
         let res = inner.version;
         inner.version += 1;
         res
@@ -345,84 +340,22 @@ where
     }
 
     fn wake_all(&self) {
-        for waker in self.wakers.iter() {
+        for waker in self.wakers.values() {
             waker.wake_by_ref();
         }
     }
 }
 
-// future
-
-pub struct ExclusiveGuard<A: BufferAllocator> {
-    inner: ArcRwLockWriteGuard<RawRwLock, RegionInner<A>>,
-}
-
-unsafe impl<A: BufferAllocator> Send for ExclusiveGuard<A> {}
-
-impl<A: BufferAllocator> Deref for ExclusiveGuard<A> {
-    type Target = RegionInner<A>;
-
-    fn deref(&self) -> &Self::Target {
-        self.inner.deref()
-    }
-}
-
-impl<A: BufferAllocator> DerefMut for ExclusiveGuard<A> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.inner.deref_mut()
-    }
-}
-
-struct ExclusiveFuture<A>
-where
-    A: BufferAllocator,
-{
-    inner: Arc<RwLock<RegionInner<A>>>,
-
-    can_write: bool,
-    can_buffered_read: bool,
-    can_physical_read: bool,
-
-    is_waker_set: bool,
-}
-
-impl<A: BufferAllocator> ExclusiveFuture<A> {
-    fn is_ready(&self, guard: &ArcRwLockWriteGuard<RawRwLock, RegionInner<A>>) -> bool {
-        tracing::trace!("exclusive: [can write: {}, writers: {}] [can buffered read: {}, buffered readers: {}] [can physical read: {}, physical readers: {}]", self.can_write, guard.writers, self.can_buffered_read, guard.buffered_readers, self.can_physical_read, guard.physical_readers);
-        (self.can_write || guard.writers == 0)
-            && (self.can_buffered_read || guard.buffered_readers == 0)
-            && (self.can_physical_read || guard.physical_readers == 0)
-    }
-}
-
-impl<A: BufferAllocator> Future for ExclusiveFuture<A> {
-    type Output = ExclusiveGuard<A>;
-
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let inner = self.inner.clone();
-        let mut guard = inner.write_arc();
-        let is_ready = self.is_ready(&guard);
-        if is_ready {
-            Poll::Ready(ExclusiveGuard { inner: guard })
-        } else {
-            if !self.is_waker_set {
-                self.is_waker_set = true;
-                guard.wakers.push(cx.waker().clone());
-            }
-            Poll::Pending
-        }
-    }
-}
 // read & write slice
 
-pub type DropCallback = Box<dyn Fn() + Send + Sync + 'static>;
-
+#[pin_project::pin_project(PinnedDrop)]
 pub struct WriteSlice {
     slice: SliceMut,
     region_id: RegionId,
     version: Version,
     offset: usize,
-    callback: DropCallback,
+    #[pin]
+    future: Option<BoxFuture<'static, ()>>,
 }
 
 impl Debug for WriteSlice {
@@ -456,6 +389,15 @@ impl WriteSlice {
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
+
+    /// # Safety
+    ///
+    /// `destroy` MUST be called before actually drop if `future` is set.
+    pub async fn destroy(mut self) {
+        if let Some(future) = self.future.take() {
+            future.await;
+        }
+    }
 }
 
 impl AsRef<[u8]> for WriteSlice {
@@ -470,12 +412,16 @@ impl AsMut<[u8]> for WriteSlice {
     }
 }
 
-impl Drop for WriteSlice {
-    fn drop(&mut self) {
-        (self.callback)();
+#[pin_project::pinned_drop]
+impl PinnedDrop for WriteSlice {
+    fn drop(self: Pin<&mut Self>) {
+        if self.future.is_some() {
+            panic!("future is not consumed: {:?}", self);
+        }
     }
 }
 
+#[pin_project::pin_project(project = ReadSliceProj, PinnedDrop)]
 pub enum ReadSlice<A>
 where
     A: BufferAllocator,
@@ -483,11 +429,13 @@ where
     Slice {
         slice: Slice,
         allocator: Option<A>,
-        callback: DropCallback,
+        #[pin]
+        future: Option<BoxFuture<'static, ()>>,
     },
     Owned {
         buf: Option<Vec<u8, A>>,
-        callback: DropCallback,
+        #[pin]
+        future: Option<BoxFuture<'static, ()>>,
     },
 }
 
@@ -523,6 +471,18 @@ where
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
+
+    /// # Safety
+    ///
+    /// `destroy` MUST be called before actually drop if `future` is set.
+    pub async fn destroy(mut self) {
+        if let Some(future) = match &mut self {
+            ReadSlice::Slice { future, .. } => future.take(),
+            ReadSlice::Owned { future, .. } => future.take(),
+        } {
+            future.await;
+        }
+    }
 }
 
 impl<A> AsRef<[u8]> for ReadSlice<A>
@@ -537,14 +497,71 @@ where
     }
 }
 
-impl<A> Drop for ReadSlice<A>
+#[pin_project::pinned_drop]
+impl<A> PinnedDrop for ReadSlice<A>
 where
     A: BufferAllocator,
 {
-    fn drop(&mut self) {
+    fn drop(self: Pin<&mut Self>) {
+        let mut this = self.project();
+        if match &mut this {
+            ReadSliceProj::Slice { future, .. } => future.is_some(),
+            ReadSliceProj::Owned { future, .. } => future.is_some(),
+        } {
+            panic!("future is not consumed: {:?}", this);
+        }
+    }
+}
+
+impl<'pin, A: BufferAllocator> Debug for ReadSliceProj<'pin, A> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            Self::Slice { callback, .. } => (callback)(),
-            Self::Owned { callback, .. } => (callback)(),
+            Self::Slice {
+                slice, allocator, ..
+            } => f
+                .debug_struct("Slice")
+                .field("slice", slice)
+                .field("allocator", allocator)
+                .finish(),
+            Self::Owned { buf, .. } => f.debug_struct("Owned").field("buf", buf).finish(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ErwLock<A: BufferAllocator> {
+    inner: Arc<RwLock<RegionInner<A>>>,
+}
+
+impl<A: BufferAllocator> ErwLock<A> {
+    pub fn new(inner: RegionInner<A>) -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(inner)),
+        }
+    }
+
+    pub async fn read(&self) -> RwLockReadGuard<'_, RegionInner<A>> {
+        self.inner.read().await
+    }
+
+    pub async fn write(&self) -> RwLockWriteGuard<'_, RegionInner<A>> {
+        self.inner.write().await
+    }
+
+    pub async fn exclusive(
+        &self,
+        can_write: bool,
+        can_buffered_read: bool,
+        can_physical_read: bool,
+    ) -> OwnedRwLockWriteGuard<RegionInner<A>> {
+        loop {
+            let guard = self.inner.clone().write_owned().await;
+            let is_ready = (can_write || guard.writers == 0)
+                && (can_buffered_read || guard.buffered_readers == 0)
+                && (can_physical_read || guard.physical_readers == 0);
+            if is_ready {
+                return guard;
+            }
         }
     }
 }
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 91ac7d87..3b7399a9 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -20,7 +20,8 @@ use foyer_intrusive::{
     eviction::EvictionPolicy,
     intrusive_adapter, key_adapter,
 };
-use tokio::sync::RwLock;
+use parking_lot::RwLock;
+use tokio::sync::RwLock as AsyncRwLock;
 
 use crate::{
     device::{BufferAllocator, Device},
@@ -41,16 +42,8 @@ where
 intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEpItem<L> { link: L } where L: Link }
 key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
 
-struct RegionManagerInner<E, EL>
-where
-    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    EL: Link,
-{
+struct RegionManagerInner {
     current: Option<RegionId>,
-
-    eviction: E,
-
-    _marker: PhantomData<EL>,
 }
 
 pub struct RegionManager<A, D, E, EL>
@@ -60,7 +53,7 @@ where
     E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
     EL: Link,
 {
-    inner: Arc<RwLock<RegionManagerInner<E, EL>>>,
+    inner: Arc<AsyncRwLock<RegionManagerInner>>,
 
     buffers: Arc<AsyncQueue<Vec<u8, A>>>,
     clean_regions: Arc<AsyncQueue<RegionId>>,
@@ -70,6 +63,9 @@ where
 
     flusher: Arc<Flusher>,
     reclaimer: Arc<Reclaimer>,
+
+    eviction: RwLock<E>,
+    _marker: PhantomData<EL>,
 }
 
 impl<A, D, E, EL> RegionManager<A, D, E, EL>
@@ -104,33 +100,33 @@ where
             items.push(item);
         }
 
-        let inner = RegionManagerInner {
-            current: None,
-
-            eviction,
-            _marker: PhantomData,
-        };
+        let inner = RegionManagerInner { current: None };
 
         Self {
-            inner: Arc::new(RwLock::new(inner)),
+            inner: Arc::new(AsyncRwLock::new(inner)),
             buffers,
             clean_regions,
             regions,
             items,
             flusher,
             reclaimer,
+            eviction: RwLock::new(eviction),
+            _marker: PhantomData,
         }
     }
 
     /// Allocate a buffer slice with given size in an active region to write.
+    #[tracing::instrument(skip(self))]
     pub async fn allocate(&self, size: usize) -> AllocateResult {
         let mut inner = self.inner.write().await;
 
         // try allocate from current region
         if let Some(region_id) = inner.current {
             let region = self.region(&region_id);
-            match region.allocate(size) {
-                AllocateResult::Ok(slice) => return AllocateResult::Ok(slice),
+            match region.allocate(size).await {
+                AllocateResult::Ok(slice) => {
+                    return AllocateResult::Ok(slice);
+                }
                 AllocateResult::Full { slice, remain } => {
                     // current region is full, schedule flushing
                     self.flusher.submit(FlushTask { region_id }).await.unwrap();
@@ -144,7 +140,8 @@ where
 
         tracing::debug!("clean regions: {}", self.clean_regions.len());
         if self.clean_regions.len() < self.reclaimer.runners() {
-            if let Some(item) = inner.eviction.pop() {
+            let item = self.eviction.write().pop();
+            if let Some(item) = item {
                 self.reclaimer
                     .submit(ReclaimTask { region_id: item.id })
                     .await
@@ -157,36 +154,54 @@ where
 
         let region = self.region(&region_id);
         let buffer = self.buffers.acquire().await;
-        region.attach_buffer(buffer);
+        region.attach_buffer(buffer).await;
 
-        let slice = region.allocate(size).unwrap();
+        let slice = region.allocate(size).await.unwrap();
 
-        region.advance();
+        region.advance().await;
         inner.current = Some(region_id);
 
         AllocateResult::Ok(slice)
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn region(&self, id: &RegionId) -> &Region<A, D> {
         &self.regions[*id as usize]
     }
 
-    pub async fn record_access(&self, id: &RegionId) {
-        let mut inner = self.inner.write().await;
+    #[tracing::instrument(skip(self))]
+    pub fn record_access(&self, id: &RegionId) {
+        let mut eviction = self.eviction.write();
         let item = &self.items[*id as usize];
         if item.link.is_linked() {
-            inner.eviction.access(&self.items[*id as usize]);
+            eviction.access(&self.items[*id as usize]);
         }
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn set_region_evictable(&self, id: &RegionId) {
-        let mut inner = self.inner.write().await;
-        let item = &self.items[*id as usize];
-        if !item.link.is_linked() {
-            inner.eviction.push(item.clone());
+        let to_reclaim = {
+            let mut eviction = self.eviction.write();
+            let item = &self.items[*id as usize];
+            if !item.link.is_linked() {
+                eviction.push(item.clone());
+            }
+            if self.clean_regions.len() < self.reclaimer.runners() {
+                Some(eviction.pop().unwrap())
+            } else {
+                None
+            }
+        };
+
+        if let Some(item) = to_reclaim {
+            self.reclaimer
+                .submit(ReclaimTask { region_id: item.id })
+                .await
+                .unwrap();
         }
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn clean_regions(&self) -> &AsyncQueue<RegionId> {
         &self.clean_regions
     }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index cc7d2115..5089840c 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -176,6 +176,7 @@ where
                     padding: remain as u64,
                 };
                 footer.write(slice.as_mut());
+                slice.destroy().await;
                 self.region_manager.allocate(serialized_len).await.unwrap()
             }
         };
@@ -193,7 +194,7 @@ where
             key,
         };
 
-        drop(slice);
+        slice.destroy().await;
 
         self.indices.insert(index);
 
@@ -206,7 +207,7 @@ where
             None => return Ok(None),
         };
 
-        self.region_manager.record_access(&index.region).await;
+        self.region_manager.record_access(&index.region);
         let region = self.region_manager.region(&index.region);
         let start = index.offset as usize;
         let end = start + index.len as usize;
@@ -217,10 +218,12 @@ where
             None => return Ok(None),
         };
 
-        match read_entry::<K, V>(slice.as_ref()) {
+        let res = match read_entry::<K, V>(slice.as_ref()) {
             Some((_key, value)) => Ok(Some(value)),
             None => Ok(None),
-        }
+        };
+        slice.destroy().await;
+        res
     }
 
     pub fn remove(&self, key: &K) {

From 7fc85420505e8e28377456f155eb5711886858c7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 4 Jul 2023 15:23:26 +0800
Subject: [PATCH 032/261] chore: ignore Cargo.lock and bump toolchain (#46)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      |   16 +-
 .github/workflows/main.yml         |   16 +-
 .github/workflows/pull-request.yml |   16 +-
 .gitignore                         |    2 +
 Cargo.lock                         | 1840 ----------------------------
 rust-toolchain                     |    3 +-
 6 files changed, 28 insertions(+), 1865 deletions(-)
 delete mode 100644 Cargo.lock

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 88b1be1c..912afa7b 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -3,10 +3,9 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-04-07
+  RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230518
-  RUNKV_CI: true
+  CACHE_KEY_SUFFIX: 20230703
 
 jobs:
   misc-check:
@@ -47,7 +46,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Install cargo-sort
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -60,8 +59,9 @@ jobs:
           cargo fmt --all -- --check
       - name: Run rust clippy check
         run: |
-          # If new CI checks are added, the one with `--locked` must be run first.
-          cargo clippy --all-targets --locked -- -D warnings
+          cargo clippy --all-targets --features tokio-console -- -D warnings &&
+          cargo clippy --all-targets --features deadlock -- -D warnings &&
+          cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
@@ -90,7 +90,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -121,7 +121,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ce135f8c..4dd59d3a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,10 +11,9 @@ on:
     branches: [main]
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2023-04-07
+  RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230518
-  RUNKV_CI: true
+  CACHE_KEY_SUFFIX: 20230703
 jobs:
   misc-check:
     name: misc check
@@ -54,7 +53,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Install cargo-sort
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -67,8 +66,9 @@ jobs:
           cargo fmt --all -- --check
       - name: Run rust clippy check
         run: |
-          # If new CI checks are added, the one with `--locked` must be run first.
-          cargo clippy --all-targets --locked -- -D warnings
+          cargo clippy --all-targets --features tokio-console -- -D warnings &&
+          cargo clippy --all-targets --features deadlock -- -D warnings &&
+          cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
@@ -96,7 +96,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -126,7 +126,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index b4beb2af..c949d49e 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -10,10 +10,9 @@ on:
   pull_request:
     branches: [main]
 env:
-  RUST_TOOLCHAIN: nightly-2023-04-07
+  RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230518
-  RUNKV_CI: true
+  CACHE_KEY_SUFFIX: 20230703
 jobs:
   misc-check:
     name: misc check
@@ -53,7 +52,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Install cargo-sort
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -66,8 +65,9 @@ jobs:
           cargo fmt --all -- --check
       - name: Run rust clippy check
         run: |
-          # If new CI checks are added, the one with `--locked` must be run first.
-          cargo clippy --all-targets --locked -- -D warnings
+          cargo clippy --all-targets --features tokio-console -- -D warnings &&
+          cargo clippy --all-targets --features deadlock -- -D warnings &&
+          cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
@@ -95,7 +95,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -125,7 +125,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
diff --git a/.gitignore b/.gitignore
index e040625b..22d3441b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 /.vscode
 
 /target
+
+Cargo.lock
diff --git a/Cargo.lock b/Cargo.lock
deleted file mode 100644
index 86daa791..00000000
--- a/Cargo.lock
+++ /dev/null
@@ -1,1840 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "addr2line"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
-dependencies = [
- "gimli",
-]
-
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
-[[package]]
-name = "anstream"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is-terminal",
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
-
-[[package]]
-name = "anstyle-parse"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
-dependencies = [
- "windows-sys",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
-dependencies = [
- "anstyle",
- "windows-sys",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.71"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
-
-[[package]]
-name = "async-channel"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"
-dependencies = [
- "concurrent-queue",
- "event-listener",
- "futures-core",
-]
-
-[[package]]
-name = "async-trait"
-version = "0.1.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b2d0f03b3640e3a630367e40c468cb7f309529c708ed1d88597047b0e7c6ef7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "axum"
-version = "0.6.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39"
-dependencies = [
- "async-trait",
- "axum-core",
- "bitflags 1.3.2",
- "bytes",
- "futures-util",
- "http",
- "http-body",
- "hyper",
- "itoa",
- "matchit",
- "memchr",
- "mime",
- "percent-encoding",
- "pin-project-lite",
- "rustversion",
- "serde",
- "sync_wrapper",
- "tower",
- "tower-layer",
- "tower-service",
-]
-
-[[package]]
-name = "axum-core"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
-dependencies = [
- "async-trait",
- "bytes",
- "futures-util",
- "http",
- "http-body",
- "mime",
- "rustversion",
- "tower-layer",
- "tower-service",
-]
-
-[[package]]
-name = "backtrace"
-version = "0.3.68"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
-dependencies = [
- "addr2line",
- "cc",
- "cfg-if",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
-[[package]]
-name = "base64"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
-
-[[package]]
-name = "base64"
-version = "0.21.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "bitflags"
-version = "2.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
-
-[[package]]
-name = "byteorder"
-version = "1.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
-
-[[package]]
-name = "bytes"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
-
-[[package]]
-name = "bytesize"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38fcc2979eff34a4b84e1cf9a1e3da42a7d44b3b690a40cdcb23e3d556cfb2e5"
-
-[[package]]
-name = "cc"
-version = "1.0.79"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "clap"
-version = "4.3.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a"
-dependencies = [
- "clap_builder",
- "clap_derive",
- "once_cell",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.3.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim",
-]
-
-[[package]]
-name = "clap_derive"
-version = "4.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
-
-[[package]]
-name = "cmsketch"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "467e460587e81453bf9aeb43cd534e9c5ad670042023bd6c3f377c23b76cc2f0"
-dependencies = [
- "paste",
-]
-
-[[package]]
-name = "colorchoice"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
-
-[[package]]
-name = "concurrent-queue"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "console-api"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e"
-dependencies = [
- "prost",
- "prost-types",
- "tonic",
- "tracing-core",
-]
-
-[[package]]
-name = "console-subscriber"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ab2224a0311582eb03adba4caaf18644f7b1f10a760803a803b9b605187fc7"
-dependencies = [
- "console-api",
- "crossbeam-channel",
- "crossbeam-utils",
- "futures",
- "hdrhistogram",
- "humantime",
- "prost-types",
- "serde",
- "serde_json",
- "thread_local",
- "tokio",
- "tokio-stream",
- "tonic",
- "tracing",
- "tracing-core",
- "tracing-subscriber",
-]
-
-[[package]]
-name = "crc32fast"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "crossbeam"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c"
-dependencies = [
- "cfg-if",
- "crossbeam-channel",
- "crossbeam-deque",
- "crossbeam-epoch",
- "crossbeam-queue",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
-dependencies = [
- "cfg-if",
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
-dependencies = [
- "autocfg",
- "cfg-if",
- "crossbeam-utils",
- "memoffset 0.9.0",
- "scopeguard",
-]
-
-[[package]]
-name = "crossbeam-queue"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "either"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
-
-[[package]]
-name = "errno"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
-dependencies = [
- "errno-dragonfly",
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
- "libc",
-]
-
-[[package]]
-name = "event-listener"
-version = "2.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
-
-[[package]]
-name = "fastrand"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
-dependencies = [
- "instant",
-]
-
-[[package]]
-name = "fixedbitset"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
-
-[[package]]
-name = "flate2"
-version = "1.0.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
-dependencies = [
- "crc32fast",
- "miniz_oxide",
-]
-
-[[package]]
-name = "fnv"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
-
-[[package]]
-name = "foyer"
-version = "0.1.0"
-dependencies = [
- "async-trait",
- "bytes",
- "bytesize",
- "clap",
- "cmsketch",
- "crossbeam",
- "foyer-common",
- "foyer-intrusive",
- "foyer-storage",
- "futures",
- "hdrhistogram",
- "itertools",
- "libc",
- "memoffset 0.8.0",
- "nix",
- "parking_lot",
- "paste",
- "prometheus",
- "rand",
- "rand_mt",
- "tempfile",
- "thiserror",
- "tokio",
- "tracing",
- "twox-hash",
-]
-
-[[package]]
-name = "foyer-common"
-version = "0.1.0"
-dependencies = [
- "bytes",
- "paste",
- "tokio",
-]
-
-[[package]]
-name = "foyer-intrusive"
-version = "0.1.0"
-dependencies = [
- "bytes",
- "cmsketch",
- "foyer-common",
- "itertools",
- "memoffset 0.8.0",
- "parking_lot",
- "paste",
- "thiserror",
- "tracing",
- "twox-hash",
-]
-
-[[package]]
-name = "foyer-storage"
-version = "0.1.0"
-dependencies = [
- "async-channel",
- "async-trait",
- "bitflags 2.3.3",
- "bytes",
- "bytesize",
- "clap",
- "cmsketch",
- "foyer-common",
- "foyer-intrusive",
- "futures",
- "hdrhistogram",
- "itertools",
- "libc",
- "memoffset 0.8.0",
- "nix",
- "parking_lot",
- "paste",
- "pin-project",
- "prometheus",
- "rand",
- "rand_mt",
- "tempfile",
- "thiserror",
- "tokio",
- "tracing",
- "twox-hash",
-]
-
-[[package]]
-name = "foyer-storage-bench"
-version = "0.1.0"
-dependencies = [
- "bytesize",
- "clap",
- "console-subscriber",
- "foyer-intrusive",
- "foyer-storage",
- "futures",
- "hdrhistogram",
- "itertools",
- "libc",
- "nix",
- "parking_lot",
- "rand",
- "rand_mt",
- "tempfile",
- "tokio",
- "tracing",
- "tracing-subscriber",
-]
-
-[[package]]
-name = "futures"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-executor",
- "futures-io",
- "futures-sink",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-channel"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
-dependencies = [
- "futures-core",
- "futures-sink",
-]
-
-[[package]]
-name = "futures-core"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
-
-[[package]]
-name = "futures-executor"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
-dependencies = [
- "futures-core",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-io"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
-
-[[package]]
-name = "futures-macro"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "futures-sink"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"
-
-[[package]]
-name = "futures-task"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
-
-[[package]]
-name = "futures-util"
-version = "0.3.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-io",
- "futures-macro",
- "futures-sink",
- "futures-task",
- "memchr",
- "pin-project-lite",
- "pin-utils",
- "slab",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
-[[package]]
-name = "gimli"
-version = "0.27.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
-
-[[package]]
-name = "h2"
-version = "0.3.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049"
-dependencies = [
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "futures-util",
- "http",
- "indexmap",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-
-[[package]]
-name = "hdrhistogram"
-version = "7.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8"
-dependencies = [
- "base64 0.13.1",
- "byteorder",
- "crossbeam-channel",
- "flate2",
- "nom",
- "num-traits",
-]
-
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
-[[package]]
-name = "hermit-abi"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
-
-[[package]]
-name = "http"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
-dependencies = [
- "bytes",
- "fnv",
- "itoa",
-]
-
-[[package]]
-name = "http-body"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
-dependencies = [
- "bytes",
- "http",
- "pin-project-lite",
-]
-
-[[package]]
-name = "httparse"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
-
-[[package]]
-name = "httpdate"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
-
-[[package]]
-name = "humantime"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
-
-[[package]]
-name = "hyper"
-version = "0.14.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "httparse",
- "httpdate",
- "itoa",
- "pin-project-lite",
- "socket2",
- "tokio",
- "tower-service",
- "tracing",
- "want",
-]
-
-[[package]]
-name = "hyper-timeout"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
-dependencies = [
- "hyper",
- "pin-project-lite",
- "tokio",
- "tokio-io-timeout",
-]
-
-[[package]]
-name = "indexmap"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
-dependencies = [
- "autocfg",
- "hashbrown",
-]
-
-[[package]]
-name = "instant"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "io-lifetimes"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
-dependencies = [
- "hermit-abi",
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "is-terminal"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb"
-dependencies = [
- "hermit-abi",
- "rustix 0.38.2",
- "windows-sys",
-]
-
-[[package]]
-name = "itertools"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0aa48fab2893d8a49caa94082ae8488f4e1050d73b367881dcd2198f4199fd8"
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
-name = "libc"
-version = "0.2.147"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
-
-[[package]]
-name = "lock_api"
-version = "0.4.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
-dependencies = [
- "autocfg",
- "scopeguard",
-]
-
-[[package]]
-name = "log"
-version = "0.4.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
-
-[[package]]
-name = "matchers"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
-dependencies = [
- "regex-automata",
-]
-
-[[package]]
-name = "matchit"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
-
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
-[[package]]
-name = "memoffset"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "memoffset"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "memoffset"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "mime"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
-[[package]]
-name = "miniz_oxide"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
-dependencies = [
- "adler",
-]
-
-[[package]]
-name = "mio"
-version = "0.8.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
-dependencies = [
- "libc",
- "wasi",
- "windows-sys",
-]
-
-[[package]]
-name = "nix"
-version = "0.26.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
-dependencies = [
- "bitflags 1.3.2",
- "cfg-if",
- "libc",
- "memoffset 0.7.1",
- "pin-utils",
- "static_assertions",
-]
-
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
-[[package]]
-name = "nu-ansi-term"
-version = "0.46.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
-dependencies = [
- "overload",
- "winapi",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "num_cpus"
-version = "1.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
-dependencies = [
- "hermit-abi",
- "libc",
-]
-
-[[package]]
-name = "object"
-version = "0.31.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
-
-[[package]]
-name = "overload"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
-dependencies = [
- "backtrace",
- "cfg-if",
- "libc",
- "petgraph",
- "redox_syscall 0.3.5",
- "smallvec",
- "thread-id",
- "windows-targets",
-]
-
-[[package]]
-name = "paste"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
-
-[[package]]
-name = "percent-encoding"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
-
-[[package]]
-name = "petgraph"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
-dependencies = [
- "fixedbitset",
- "indexmap",
-]
-
-[[package]]
-name = "pin-project"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842"
-dependencies = [
- "pin-project-internal",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "pin-project-lite"
-version = "0.2.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
-
-[[package]]
-name = "pin-utils"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.63"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "prometheus"
-version = "0.13.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
-dependencies = [
- "cfg-if",
- "fnv",
- "lazy_static",
- "memchr",
- "parking_lot",
- "protobuf",
- "thiserror",
-]
-
-[[package]]
-name = "prost"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
-dependencies = [
- "bytes",
- "prost-derive",
-]
-
-[[package]]
-name = "prost-derive"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
-dependencies = [
- "anyhow",
- "itertools",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "prost-types"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
-dependencies = [
- "prost",
-]
-
-[[package]]
-name = "protobuf"
-version = "2.28.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
-
-[[package]]
-name = "quote"
-version = "1.0.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "rand_mt"
-version = "4.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49e018c6ded60e5252609887c12eb3ca2592e9248c5894a7db3975c8a7a1e2df"
-dependencies = [
- "rand_core",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags 1.3.2",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
-dependencies = [
- "bitflags 1.3.2",
-]
-
-[[package]]
-name = "regex"
-version = "1.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
-dependencies = [
- "regex-syntax 0.7.2",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
-dependencies = [
- "regex-syntax 0.6.29",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.6.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
-
-[[package]]
-name = "regex-syntax"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
-
-[[package]]
-name = "rustc-demangle"
-version = "0.1.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
-
-[[package]]
-name = "rustix"
-version = "0.37.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8818fa822adcc98b18fedbb3632a6a33213c070556b5aa7c4c8cc21cff565c4c"
-dependencies = [
- "bitflags 1.3.2",
- "errno",
- "io-lifetimes",
- "libc",
- "linux-raw-sys 0.3.8",
- "windows-sys",
-]
-
-[[package]]
-name = "rustix"
-version = "0.38.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4"
-dependencies = [
- "bitflags 2.3.3",
- "errno",
- "libc",
- "linux-raw-sys 0.4.3",
- "windows-sys",
-]
-
-[[package]]
-name = "rustversion"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"
-
-[[package]]
-name = "ryu"
-version = "1.0.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
-
-[[package]]
-name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
-[[package]]
-name = "serde"
-version = "1.0.165"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c939f902bb7d0ccc5bce4f03297e161543c2dcb30914faf032c2bd0b7a0d48fc"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.165"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6eaae920e25fffe4019b75ff65e7660e72091e59dd204cb5849bbd6a3fd343d7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.99"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
-dependencies = [
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "sharded-slab"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
-dependencies = [
- "lazy_static",
-]
-
-[[package]]
-name = "signal-hook-registry"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "slab"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
-
-[[package]]
-name = "socket2"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
-dependencies = [
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
-[[package]]
-name = "strsim"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
-
-[[package]]
-name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "sync_wrapper"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
-
-[[package]]
-name = "tempfile"
-version = "3.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
-dependencies = [
- "autocfg",
- "cfg-if",
- "fastrand",
- "redox_syscall 0.3.5",
- "rustix 0.37.22",
- "windows-sys",
-]
-
-[[package]]
-name = "thiserror"
-version = "1.0.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "thread-id"
-version = "4.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ee93aa2b8331c0fec9091548843f2c90019571814057da3b783f9de09349d73"
-dependencies = [
- "libc",
- "redox_syscall 0.2.16",
- "winapi",
-]
-
-[[package]]
-name = "thread_local"
-version = "1.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
-dependencies = [
- "cfg-if",
- "once_cell",
-]
-
-[[package]]
-name = "tokio"
-version = "1.29.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
-dependencies = [
- "autocfg",
- "backtrace",
- "bytes",
- "libc",
- "mio",
- "num_cpus",
- "pin-project-lite",
- "signal-hook-registry",
- "socket2",
- "tokio-macros",
- "tracing",
- "windows-sys",
-]
-
-[[package]]
-name = "tokio-io-timeout"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
-dependencies = [
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
-name = "tokio-macros"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "tokio-stream"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
-dependencies = [
- "futures-core",
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
-name = "tokio-util"
-version = "0.7.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-sink",
- "pin-project-lite",
- "tokio",
- "tracing",
-]
-
-[[package]]
-name = "tonic"
-version = "0.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
-dependencies = [
- "async-trait",
- "axum",
- "base64 0.21.2",
- "bytes",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "hyper",
- "hyper-timeout",
- "percent-encoding",
- "pin-project",
- "prost",
- "tokio",
- "tokio-stream",
- "tower",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "tower"
-version = "0.4.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
-dependencies = [
- "futures-core",
- "futures-util",
- "indexmap",
- "pin-project",
- "pin-project-lite",
- "rand",
- "slab",
- "tokio",
- "tokio-util",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "tower-layer"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
-
-[[package]]
-name = "tower-service"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
-
-[[package]]
-name = "tracing"
-version = "0.1.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
-dependencies = [
- "cfg-if",
- "pin-project-lite",
- "tracing-attributes",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-attributes"
-version = "0.1.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "tracing-core"
-version = "0.1.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
-dependencies = [
- "once_cell",
- "valuable",
-]
-
-[[package]]
-name = "tracing-log"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
-dependencies = [
- "lazy_static",
- "log",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-subscriber"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
-dependencies = [
- "matchers",
- "nu-ansi-term",
- "once_cell",
- "regex",
- "sharded-slab",
- "smallvec",
- "thread_local",
- "tracing",
- "tracing-core",
- "tracing-log",
-]
-
-[[package]]
-name = "try-lock"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
-
-[[package]]
-name = "twox-hash"
-version = "1.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
-dependencies = [
- "cfg-if",
- "rand",
- "static_assertions",
-]
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0"
-
-[[package]]
-name = "utf8parse"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
-
-[[package]]
-name = "valuable"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
-
-[[package]]
-name = "want"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
-dependencies = [
- "try-lock",
-]
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows-sys"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.48.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
-dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
diff --git a/rust-toolchain b/rust-toolchain
index ab84e227..b0adca9d 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1 +1,2 @@
-nightly-2023-04-07
+[toolchain]
+channel = "nightly-2023-05-31"
\ No newline at end of file

From 8979084a794ba969d3ea22f29a657f33d67b1119 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 4 Jul 2023 17:08:05 +0800
Subject: [PATCH 033/261] fix: gracefullly shutdown runners (#47)

* fix: gracefullly shutdown runners

Use `Store::shutdown_runners()` to gracefully shutdown runners.`

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* log warn if send failed instead of panic

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs     |   2 +
 foyer-storage/src/flusher.rs        | 141 ++++++++++++++++------------
 foyer-storage/src/lib.rs            |   1 +
 foyer-storage/src/reclaimer.rs      | 135 +++++++++++++++-----------
 foyer-storage/src/region_manager.rs |  24 +++--
 foyer-storage/src/store.rs          |  55 +++++++++--
 6 files changed, 221 insertions(+), 137 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 44c13a69..78cea50f 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -260,6 +260,8 @@ async fn main() {
         &metrics_dump_end,
     );
     println!("\nTotal:\n{}", analysis);
+
+    store.shutdown_runners().await.unwrap();
 }
 
 async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::Receiver<()>) {
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index f8359145..b02d92c2 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -17,9 +17,9 @@ use std::sync::Arc;
 use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use itertools::Itertools;
-use tokio::sync::{
-    mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
-    Mutex,
+use tokio::{
+    sync::{broadcast, mpsc, Mutex},
+    task::JoinHandle,
 };
 
 use crate::{
@@ -38,7 +38,7 @@ pub struct FlushTask {
 struct FlusherInner {
     sequence: usize,
 
-    task_txs: Vec<UnboundedSender<FlushTask>>,
+    task_txs: Vec<mpsc::UnboundedSender<FlushTask>>,
 }
 
 pub struct Flusher {
@@ -63,7 +63,9 @@ impl Flusher {
         &self,
         buffers: Arc<AsyncQueue<Vec<u8, A>>>,
         region_manager: Arc<RegionManager<A, D, E, EL>>,
-    ) where
+        stop_rxs: Vec<broadcast::Receiver<()>>,
+    ) -> Vec<JoinHandle<()>>
+    where
         A: BufferAllocator,
         D: Device<IoBufferAllocator = A>,
         E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
@@ -73,25 +75,30 @@ impl Flusher {
 
         #[allow(clippy::type_complexity)]
         let (mut txs, rxs): (
-            Vec<UnboundedSender<FlushTask>>,
-            Vec<UnboundedReceiver<FlushTask>>,
-        ) = (0..self.runners).map(|_| unbounded_channel()).unzip();
+            Vec<mpsc::UnboundedSender<FlushTask>>,
+            Vec<mpsc::UnboundedReceiver<FlushTask>>,
+        ) = (0..self.runners).map(|_| mpsc::unbounded_channel()).unzip();
         inner.task_txs.append(&mut txs);
 
         let runners = rxs
             .into_iter()
-            .map(|rx| Runner {
-                task_rx: rx,
+            .zip_eq(stop_rxs.into_iter())
+            .map(|(task_rx, stop_rx)| Runner {
+                task_rx,
                 buffers: buffers.clone(),
                 region_manager: region_manager.clone(),
+                stop_rx,
             })
             .collect_vec();
 
+        let mut handles = vec![];
         for runner in runners {
-            tokio::spawn(async move {
+            let handle = tokio::spawn(async move {
                 runner.run().await.unwrap();
             });
+            handles.push(handle);
         }
+        handles
     }
 
     pub fn runners(&self) -> usize {
@@ -113,10 +120,12 @@ where
     E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
     EL: Link,
 {
-    task_rx: UnboundedReceiver<FlushTask>,
+    task_rx: mpsc::UnboundedReceiver<FlushTask>,
     buffers: Arc<AsyncQueue<Vec<u8, A>>>,
 
     region_manager: Arc<RegionManager<A, D, E, EL>>,
+
+    stop_rx: broadcast::Receiver<()>,
 }
 
 impl<A, D, E, EL> Runner<A, D, E, EL>
@@ -128,71 +137,79 @@ where
 {
     async fn run(mut self) -> Result<()> {
         loop {
-            if let Some(task) = self.task_rx.recv().await {
-                // TODO(MrCroxx): seal buffer
+            tokio::select! {
+                Some(task) = self.task_rx.recv() => {
+                    self.handle(task).await?;
+                }
+                _ = self.stop_rx.recv() => {
+                    tracing::info!("[flusher] exit");
+                    return Ok(())
+                }
+            }
+        }
+    }
 
-                tracing::info!("[flusher] receive flush task, region: {}", task.region_id);
+    async fn handle(&self, task: FlushTask) -> Result<()> {
+        tracing::info!("[flusher] receive flush task, region: {}", task.region_id);
 
-                let region = self.region_manager.region(&task.region_id);
+        let region = self.region_manager.region(&task.region_id);
 
-                tracing::trace!("[flusher] step 1");
+        tracing::trace!("[flusher] step 1");
 
-                {
-                    // step 1: write buffer back to device
-                    let slice = region.load(.., 0).await?.unwrap();
+        {
+            // step 1: write buffer back to device
+            let slice = region.load(.., 0).await?.unwrap();
 
-                    // wait all physical readers (from previous version) and writers done
-                    let guard = region.exclusive(false, true, false).await;
+            // wait all physical readers (from previous version) and writers done
+            let guard = region.exclusive(false, true, false).await;
 
-                    tracing::trace!("[flusher] write region {} back to device", task.region_id);
+            tracing::trace!("[flusher] write region {} back to device", task.region_id);
 
-                    let mut offset = 0;
-                    let len = region.device().io_size();
-                    while offset < region.device().region_size() {
-                        let start = offset;
-                        let end = std::cmp::min(offset + len, region.device().region_size());
+            let mut offset = 0;
+            let len = region.device().io_size();
+            while offset < region.device().region_size() {
+                let start = offset;
+                let end = std::cmp::min(offset + len, region.device().region_size());
 
-                        let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
-                        region
-                            .device()
-                            .write(s, region.id(), offset as u64, len)
-                            .await?;
-                        offset += len;
-                    }
-                    drop(guard);
-                    slice.destroy().await;
-                }
+                let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
+                region
+                    .device()
+                    .write(s, region.id(), offset as u64, len)
+                    .await?;
+                offset += len;
+            }
+            drop(guard);
+            slice.destroy().await;
+        }
+
+        tracing::trace!("[flusher] step 2");
 
-                tracing::trace!("[flusher] step 2");
+        let buffer = {
+            // step 2: detach buffer
+            let mut guard = region.exclusive(false, false, true).await;
 
-                let buffer = {
-                    // step 2: detach buffer
-                    let mut guard = region.exclusive(false, false, true).await;
+            let buffer = guard.detach_buffer();
 
-                    let buffer = guard.detach_buffer();
+            tracing::trace!(
+                "[flusher] region {}, writers: {}, buffered readers: {}, physical readers: {}",
+                region.id(),
+                guard.writers(),
+                guard.buffered_readers(),
+                guard.physical_readers()
+            );
 
-                    tracing::trace!(
-                            "[flusher] region {}, writers: {}, buffered readers: {}, physical readers: {}",
-                            region.id(),
-                            guard.writers(),
-                            guard.buffered_readers(),
-                            guard.physical_readers()
-                        );
+            drop(guard);
+            buffer
+        };
 
-                    drop(guard);
-                    buffer
-                };
+        tracing::trace!("[flusher] step 3");
 
-                tracing::trace!("[flusher] step 3");
+        // step 3: release buffer
+        self.buffers.release(buffer);
+        self.region_manager.set_region_evictable(&region.id()).await;
 
-                // step 3: release buffer
-                self.buffers.release(buffer);
-                self.region_manager.set_region_evictable(&region.id()).await;
+        tracing::info!("[flusher] finish flush task, region: {}", task.region_id);
 
-                tracing::info!("[flusher] finish flush task, region: {}", task.region_id);
-            } else {
-                return Ok(());
-            }
-        }
+        Ok(())
     }
 }
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index b007d557..118b11af 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -16,6 +16,7 @@
 #![feature(strict_provenance)]
 #![feature(trait_alias)]
 #![feature(get_mut_unchecked)]
+#![feature(let_chains)]
 #![allow(clippy::type_complexity)]
 
 use device::io_buffer::AlignedAllocator;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 7e5da64c..de8a7535 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -31,9 +31,9 @@ use foyer_common::{
 };
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use itertools::Itertools;
-use tokio::sync::{
-    mpsc::{channel, Receiver, Sender},
-    Mutex,
+use tokio::{
+    sync::{broadcast, mpsc, Mutex},
+    task::JoinHandle,
 };
 
 #[derive(Debug)]
@@ -44,7 +44,7 @@ pub struct ReclaimTask {
 struct ReclaimerInner {
     sequence: usize,
 
-    task_txs: Vec<Sender<ReclaimTask>>,
+    task_txs: Vec<mpsc::Sender<ReclaimTask>>,
 }
 
 pub struct Reclaimer {
@@ -73,7 +73,9 @@ impl Reclaimer {
         clean_regions: Arc<AsyncQueue<RegionId>>,
         reinsertion: RP,
         indices: Arc<Indices<K>>,
-    ) where
+        stop_rxs: Vec<broadcast::Receiver<()>>,
+    ) -> Vec<JoinHandle<()>>
+    where
         K: Key,
         V: Value,
         A: BufferAllocator,
@@ -86,27 +88,34 @@ impl Reclaimer {
         let mut inner = self.inner.lock().await;
 
         #[allow(clippy::type_complexity)]
-        let (mut txs, rxs): (Vec<Sender<ReclaimTask>>, Vec<Receiver<ReclaimTask>>) =
-            (0..self.runners).map(|_| channel(1)).unzip();
+        let (mut txs, rxs): (
+            Vec<mpsc::Sender<ReclaimTask>>,
+            Vec<mpsc::Receiver<ReclaimTask>>,
+        ) = (0..self.runners).map(|_| mpsc::channel(1)).unzip();
         inner.task_txs.append(&mut txs);
 
         let runners = rxs
             .into_iter()
-            .map(|rx| Runner {
-                task_rx: rx,
+            .zip_eq(stop_rxs.into_iter())
+            .map(|(task_rx, stop_rx)| Runner {
+                task_rx,
                 _store: store.clone(),
                 region_manager: region_manager.clone(),
                 clean_regions: clean_regions.clone(),
                 _reinsertion: reinsertion.clone(),
                 indices: indices.clone(),
+                stop_rx,
             })
             .collect_vec();
 
+        let mut handles = vec![];
         for runner in runners {
-            tokio::spawn(async move {
+            let handle = tokio::spawn(async move {
                 runner.run().await.unwrap();
             });
+            handles.push(handle);
         }
+        handles
     }
 
     pub fn runners(&self) -> usize {
@@ -135,13 +144,15 @@ where
     RP: ReinsertionPolicy<Key = K, Value = V>,
     EL: Link,
 {
-    task_rx: Receiver<ReclaimTask>,
+    task_rx: mpsc::Receiver<ReclaimTask>,
 
     _store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
     region_manager: Arc<RegionManager<A, D, EP, EL>>,
     clean_regions: Arc<AsyncQueue<RegionId>>,
     _reinsertion: RP,
     indices: Arc<Indices<K>>,
+
+    stop_rx: broadcast::Receiver<()>,
 }
 
 impl<K, V, A, D, EP, AP, RP, EL> Runner<K, V, A, D, EP, AP, RP, EL>
@@ -157,53 +168,63 @@ where
 {
     async fn run(mut self) -> Result<()> {
         loop {
-            if let Some(task) = self.task_rx.recv().await {
-                tracing::info!(
-                    "[reclaimer] receive reclaim task, region: {}",
-                    task.region_id
-                );
-
-                let region = self.region_manager.region(&task.region_id);
-
-                // keep region totally exclusive while reclamation
-                let guard = region.exclusive(false, false, false).await;
-
-                tracing::trace!(
-                    "[reclaimer] region {}, writers: {}, buffered readers: {}, physical readers: {}",
-                    region.id(),
-                    guard.writers(),
-                    guard.buffered_readers(),
-                    guard.physical_readers()
-                );
-
-                // step 1: drop indices
-                let _indices = self.indices.take_region(&task.region_id);
-
-                // step 2: do reinsertion
-                // TODO(MrCroxx): do reinsertion
-
-                // step 3: set region last block zero
-                let align = region.device().align();
-                let region_size = region.device().region_size();
-                let mut buf = region.device().io_buffer(align, align);
-                (&mut buf[..]).put_slice(&vec![0; align]);
-                region
-                    .device()
-                    .write(buf, task.region_id, (region_size - align) as u64, align)
-                    .await?;
-
-                // step 4: send clean region
-                self.clean_regions.release(task.region_id);
-
-                drop(guard);
-
-                tracing::info!(
-                    "[reclaimer] finish reclaim task, region: {}",
-                    task.region_id
-                );
-            } else {
-                return Ok(());
+            tokio::select! {
+                Some(task) = self.task_rx.recv() => {
+                    self.handle(task).await?;
+                }
+                _ = self.stop_rx.recv() => {
+                    tracing::info!("[reclaimer] exit");
+                    return Ok(())
+                }
             }
         }
     }
+
+    async fn handle(&self, task: ReclaimTask) -> Result<()> {
+        tracing::info!(
+            "[reclaimer] receive reclaim task, region: {}",
+            task.region_id
+        );
+
+        let region = self.region_manager.region(&task.region_id);
+
+        // keep region totally exclusive while reclamation
+        let guard = region.exclusive(false, false, false).await;
+
+        tracing::trace!(
+            "[reclaimer] region {}, writers: {}, buffered readers: {}, physical readers: {}",
+            region.id(),
+            guard.writers(),
+            guard.buffered_readers(),
+            guard.physical_readers()
+        );
+
+        // step 1: drop indices
+        let _indices = self.indices.take_region(&task.region_id);
+
+        // step 2: do reinsertion
+        // TODO(MrCroxx): do reinsertion
+
+        // step 3: set region last block zero
+        let align = region.device().align();
+        let region_size = region.device().region_size();
+        let mut buf = region.device().io_buffer(align, align);
+        (&mut buf[..]).put_slice(&vec![0; align]);
+        region
+            .device()
+            .write(buf, task.region_id, (region_size - align) as u64, align)
+            .await?;
+
+        // step 4: send clean region
+        self.clean_regions.release(task.region_id);
+
+        drop(guard);
+
+        tracing::info!(
+            "[reclaimer] finish reclaim task, region: {}",
+            task.region_id
+        );
+
+        Ok(())
+    }
 }
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 3b7399a9..78aaac80 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -129,7 +129,7 @@ where
                 }
                 AllocateResult::Full { slice, remain } => {
                     // current region is full, schedule flushing
-                    self.flusher.submit(FlushTask { region_id }).await.unwrap();
+                    self.submit_flush_task(FlushTask { region_id }).await;
                     inner.current = None;
                     return AllocateResult::Full { slice, remain };
                 }
@@ -142,10 +142,8 @@ where
         if self.clean_regions.len() < self.reclaimer.runners() {
             let item = self.eviction.write().pop();
             if let Some(item) = item {
-                self.reclaimer
-                    .submit(ReclaimTask { region_id: item.id })
-                    .await
-                    .unwrap();
+                self.submit_reclaim_task(ReclaimTask { region_id: item.id })
+                    .await;
             }
         }
 
@@ -194,10 +192,8 @@ where
         };
 
         if let Some(item) = to_reclaim {
-            self.reclaimer
-                .submit(ReclaimTask { region_id: item.id })
+            self.submit_reclaim_task(ReclaimTask { region_id: item.id })
                 .await
-                .unwrap();
         }
     }
 
@@ -205,4 +201,16 @@ where
     pub fn clean_regions(&self) -> &AsyncQueue<RegionId> {
         &self.clean_regions
     }
+
+    async fn submit_reclaim_task(&self, task: ReclaimTask) {
+        if let Err(e) = self.reclaimer.submit(task).await {
+            tracing::warn!("fail to submit reclaim task: {:?}", e);
+        }
+    }
+
+    async fn submit_flush_task(&self, task: FlushTask) {
+        if let Err(e) = self.flusher.submit(task).await {
+            tracing::warn!("fail to submit reclaim task: {:?}", e);
+        }
+    }
 }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 5089840c..cf38088e 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -17,6 +17,9 @@ use std::{fmt::Debug, marker::PhantomData, sync::Arc};
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, queue::AsyncQueue};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use itertools::Itertools;
+use parking_lot::Mutex;
+use tokio::{sync::broadcast, task::JoinHandle};
 use twox_hash::XxHash64;
 
 use crate::{
@@ -93,6 +96,10 @@ where
 
     admission: AP,
 
+    handles: Mutex<Vec<JoinHandle<()>>>,
+
+    stop_tx: broadcast::Sender<()>,
+
     _marker: PhantomData<(V, RP)>,
 }
 
@@ -136,30 +143,58 @@ where
 
         let indices = Arc::new(Indices::new(device.regions()));
 
+        let (stop_tx, _stop_rx) = broadcast::channel(config.flushers + config.reclaimers + 1);
+        let flusher_stop_rxs = (0..config.flushers)
+            .map(|_| stop_tx.subscribe())
+            .collect_vec();
+        let reclaimer_stop_rxs = (0..config.reclaimers)
+            .map(|_| stop_tx.subscribe())
+            .collect_vec();
+
         let store = Arc::new(Self {
             indices: indices.clone(),
             region_manager: region_manager.clone(),
             device: device.clone(),
             admission: config.admission,
+            handles: Mutex::new(vec![]),
+            stop_tx,
             _marker: PhantomData,
         });
 
         store.recover(config.recover_concurrency).await?;
 
-        flusher.run(buffers, region_manager.clone()).await;
-        reclaimer
-            .run(
-                store.clone(),
-                region_manager,
-                clean_regions,
-                config.reinsertion,
-                indices,
-            )
-            .await;
+        let mut handles = vec![];
+        handles.append(
+            &mut flusher
+                .run(buffers, region_manager.clone(), flusher_stop_rxs)
+                .await,
+        );
+        handles.append(
+            &mut reclaimer
+                .run(
+                    store.clone(),
+                    region_manager,
+                    clean_regions,
+                    config.reinsertion,
+                    indices,
+                    reclaimer_stop_rxs,
+                )
+                .await,
+        );
+        store.handles.lock().append(&mut handles);
 
         Ok(store)
     }
 
+    pub async fn shutdown_runners(&self) -> Result<()> {
+        self.stop_tx.send(()).unwrap();
+        let handles = self.handles.lock().drain(..).collect_vec();
+        for handle in handles {
+            handle.await.unwrap();
+        }
+        Ok(())
+    }
+
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
         if !self.admission.judge(&key, &value) {
             return Ok(false);

From 1018bca29a96b2a9c98a256662f7e0cf41eb9e1b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 5 Jul 2023 14:45:34 +0800
Subject: [PATCH 034/261] chore: export storage config (#50)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/lib.rs | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 118b11af..ccda3835 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -46,6 +46,16 @@ pub type LruFsStore<K, V, AP, RP> = store::Store<
     foyer_intrusive::eviction::lru::LruLink,
 >;
 
+pub type LruFsStoreConfig<AP, RP> = store::StoreConfig<
+    device::fs::FsDevice,
+    AP,
+    foyer_intrusive::eviction::lru::Lru<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
+    >,
+    RP,
+    foyer_intrusive::eviction::lru::LruLink,
+>;
+
 pub type LfuFsStore<K, V, AP, RP> = store::Store<
     K,
     V,
@@ -59,6 +69,16 @@ pub type LfuFsStore<K, V, AP, RP> = store::Store<
     foyer_intrusive::eviction::lfu::LfuLink,
 >;
 
+pub type LfuFsStoreConfig<AP, RP> = store::StoreConfig<
+    device::fs::FsDevice,
+    AP,
+    foyer_intrusive::eviction::lfu::Lfu<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
+    >,
+    RP,
+    foyer_intrusive::eviction::lfu::LfuLink,
+>;
+
 pub type FifoFsStore<K, V, AP, RP> = store::Store<
     K,
     V,
@@ -71,3 +91,13 @@ pub type FifoFsStore<K, V, AP, RP> = store::Store<
     RP,
     foyer_intrusive::eviction::fifo::FifoLink,
 >;
+
+pub type FifoFsStoreConfig<AP, RP> = store::StoreConfig<
+    device::fs::FsDevice,
+    AP,
+    foyer_intrusive::eviction::fifo::Fifo<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
+    >,
+    RP,
+    foyer_intrusive::eviction::fifo::FifoLink,
+>;

From 011a401d88e0888e95cc9073404d7ce5fecd60fe Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 5 Jul 2023 15:29:27 +0800
Subject: [PATCH 035/261] fix: AdmitAll & ReinsertNone no longer requires KV
 default (#51)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/admission.rs   | 8 +++++++-
 foyer-storage/src/reinsertion.rs | 8 +++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/foyer-storage/src/admission.rs b/foyer-storage/src/admission.rs
index f80cd3d3..b953a4f7 100644
--- a/foyer-storage/src/admission.rs
+++ b/foyer-storage/src/admission.rs
@@ -25,9 +25,15 @@ pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct AdmitAll<K: Key, V: Value>(PhantomData<(K, V)>);
 
+impl<K: Key, V: Value> Default for AdmitAll<K, V> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
 impl<K: Key, V: Value> AdmissionPolicy for AdmitAll<K, V> {
     type Key = K;
 
diff --git a/foyer-storage/src/reinsertion.rs b/foyer-storage/src/reinsertion.rs
index 96ca5b13..9928a86f 100644
--- a/foyer-storage/src/reinsertion.rs
+++ b/foyer-storage/src/reinsertion.rs
@@ -25,9 +25,15 @@ pub trait ReinsertionPolicy: Send + Sync + 'static + Clone + Debug {
     fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct ReinsertNone<K: Key, V: Value>(PhantomData<(K, V)>);
 
+impl<K: Key, V: Value> Default for ReinsertNone<K, V> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
 impl<K: Key, V: Value> Clone for ReinsertNone<K, V> {
     fn clone(&self) -> Self {
         Self(PhantomData)

From e61eb5dbe27b99ca926b1f4a6950374167b3f94f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 6 Jul 2023 13:27:46 +0800
Subject: [PATCH 036/261] fix: region advance & recovery & seal (#53)

- fix region advance
- fix recovery slice destroy
- seal last dirty region when shutdown

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/eviction/fifo.rs |   2 +-
 foyer-storage/src/flusher.rs         |   1 +
 foyer-storage/src/reclaimer.rs       |   1 +
 foyer-storage/src/region.rs          |   2 +-
 foyer-storage/src/region_manager.rs  |   8 +-
 foyer-storage/src/store.rs           | 161 ++++++++++++++++++++++++++-
 6 files changed, 168 insertions(+), 7 deletions(-)

diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 86d3d97d..846581de 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -48,7 +48,7 @@ pub struct FifoConfig {
     /// The formula is as follows:
     ///
     /// `segment's size = total_segments * (segment's ratio / sum(ratio))`
-    segment_ratios: Vec<usize>,
+    pub segment_ratios: Vec<usize>,
 }
 
 #[derive(Debug, Default)]
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index b02d92c2..7f3f96a1 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -138,6 +138,7 @@ where
     async fn run(mut self) -> Result<()> {
         loop {
             tokio::select! {
+                biased;
                 Some(task) = self.task_rx.recv() => {
                     self.handle(task).await?;
                 }
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index de8a7535..73c4af9b 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -169,6 +169,7 @@ where
     async fn run(mut self) -> Result<()> {
         loop {
             tokio::select! {
+                biased;
                 Some(task) = self.task_rx.recv() => {
                     self.handle(task).await?;
                 }
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 3615661d..7e86fac6 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -92,7 +92,7 @@ where
 {
     pub fn new(id: RegionId, device: D) -> Self {
         let inner = RegionInner {
-            version: 1,
+            version: 0,
 
             buffer: None,
             len: 0,
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 78aaac80..43803921 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -18,7 +18,7 @@ use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{
     core::{adapter::Link, pointer::PointerOps},
     eviction::EvictionPolicy,
-    intrusive_adapter, key_adapter,
+    intrusive_adapter, key_adapter, priority_adapter,
 };
 use parking_lot::RwLock;
 use tokio::sync::RwLock as AsyncRwLock;
@@ -37,10 +37,12 @@ where
 {
     link: L,
     id: RegionId,
+    priority: usize,
 }
 
 intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEpItem<L> { link: L } where L: Link }
 key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
+priority_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { priority: usize } where L: Link }
 
 struct RegionManagerInner {
     current: Option<RegionId>,
@@ -94,6 +96,7 @@ where
             let item = Arc::new(RegionEpItem {
                 link: E::Link::default(),
                 id,
+                priority: 0,
             });
 
             regions.push(region);
@@ -151,12 +154,13 @@ where
         tracing::info!("switch to clean region: {}", region_id);
 
         let region = self.region(&region_id);
+        region.advance().await;
+
         let buffer = self.buffers.acquire().await;
         region.attach_buffer(buffer).await;
 
         let slice = region.allocate(size).await.unwrap();
 
-        region.advance().await;
         inner.current = Some(region_id);
 
         AllocateResult::Ok(slice)
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index cf38088e..358d9c11 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -187,6 +187,7 @@ where
     }
 
     pub async fn shutdown_runners(&self) -> Result<()> {
+        self.seal().await?;
         self.stop_tx.send(()).unwrap();
         let handles = self.handles.lock().drain(..).collect_vec();
         for handle in handles {
@@ -271,6 +272,31 @@ where
         bits::align_up(self.device.align(), unaligned)
     }
 
+    async fn seal(&self) -> Result<()> {
+        match self
+            .region_manager
+            .allocate(self.device.region_size() - self.device.align())
+            .await
+        {
+            crate::region::AllocateResult::Full { mut slice, remain } => {
+                println!("seal");
+                // current region is full, write region footer and try allocate again
+                let footer = RegionFooter {
+                    magic: REGION_MAGIC,
+                    padding: remain as u64,
+                };
+                footer.write(slice.as_mut());
+                slice.destroy().await;
+            }
+            crate::region::AllocateResult::Ok(slice) => {
+                println!("not seal");
+                // region is empty, skip
+                slice.destroy().await
+            }
+        }
+        Ok(())
+    }
+
     async fn recover(&self, concurrency: usize) -> Result<()> {
         tracing::info!("start store recovery");
 
@@ -500,6 +526,8 @@ where
         };
 
         let footer = RegionFooter::read(slice.as_ref());
+        slice.destroy().await;
+
         if footer.magic != REGION_MAGIC {
             return Ok(None);
         }
@@ -539,13 +567,18 @@ where
             let rel_start =
                 align - EntryFooter::serialized_len() - (footer.padding + footer.key_len) as usize;
             let rel_end = align - EntryFooter::serialized_len() - footer.padding as usize;
-            K::read(&slice.as_ref()[rel_start..rel_end])
+            let key = K::read(&slice.as_ref()[rel_start..rel_end]);
+            slice.destroy().await;
+            key
         } else {
-            let slice = self.region.load(align_start..align_end, 0).await?.unwrap();
+            slice.destroy().await;
+            let s = self.region.load(align_start..align_end, 0).await?.unwrap();
             let rel_start = abs_start - align_start;
             let rel_end = abs_end - align_start;
 
-            K::read(&slice.as_ref()[rel_start..rel_end])
+            let key = K::read(&s.as_ref()[rel_start..rel_end]);
+            s.destroy().await;
+            key
         };
 
         self.cursor -= entry_len;
@@ -561,3 +594,125 @@ where
         }))
     }
 }
+
+#[cfg(test)]
+pub mod tests {
+    use std::path::PathBuf;
+
+    use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
+
+    use crate::{
+        admission::AdmitAll,
+        device::{
+            fs::{FsDevice, FsDeviceConfig},
+            io_buffer::AlignedAllocator,
+        },
+        reinsertion::ReinsertNone,
+    };
+
+    use super::*;
+
+    type TestStore = Store<
+        u64,
+        Vec<u8>,
+        AlignedAllocator,
+        FsDevice,
+        Fifo<RegionEpItemAdapter<FifoLink>>,
+        AdmitAll<u64, Vec<u8>>,
+        ReinsertNone<u64, Vec<u8>>,
+        FifoLink,
+    >;
+
+    type TestStoreConfig = StoreConfig<
+        FsDevice,
+        AdmitAll<u64, Vec<u8>>,
+        Fifo<RegionEpItemAdapter<FifoLink>>,
+        ReinsertNone<u64, Vec<u8>>,
+        FifoLink,
+    >;
+
+    #[tokio::test]
+    #[allow(clippy::identity_op)]
+    async fn test_recovery() {
+        const KB: usize = 1024;
+        const MB: usize = 1024 * 1024;
+
+        let tempdir = tempfile::tempdir().unwrap();
+
+        let config = TestStoreConfig {
+            eviction_config: FifoConfig {
+                segment_ratios: vec![1],
+            },
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            admission: AdmitAll::default(),
+            reinsertion: ReinsertNone::default(),
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            reclaimers: 1,
+            recover_concurrency: 2,
+        };
+
+        let store = TestStore::open(config).await.unwrap();
+
+        // files:
+        // [0, 1, 2] (evicted)
+        // [3, 4, 5]
+        // [6, 7, 8]
+        // [9, 10, 11]
+        for i in 0..12 {
+            store.insert(i, vec![i as u8; 1 * MB]).await.unwrap();
+        }
+
+        for i in 0..3 {
+            assert!(store.lookup(&i).await.unwrap().is_none());
+        }
+        for i in 3..12 {
+            assert_eq!(
+                store.lookup(&i).await.unwrap().unwrap(),
+                vec![i as u8; 1 * MB],
+            );
+        }
+
+        store.shutdown_runners().await.unwrap();
+        drop(store);
+
+        let config = TestStoreConfig {
+            eviction_config: FifoConfig {
+                segment_ratios: vec![1],
+            },
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            admission: AdmitAll::default(),
+            reinsertion: ReinsertNone::default(),
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            reclaimers: 1,
+            recover_concurrency: 2,
+        };
+        let store = TestStore::open(config).await.unwrap();
+
+        for i in 0..3 {
+            assert!(store.lookup(&i).await.unwrap().is_none());
+        }
+        for i in 3..12 {
+            assert_eq!(
+                store.lookup(&i).await.unwrap().unwrap(),
+                vec![i as u8; 1 * MB],
+            );
+        }
+
+        store.shutdown_runners().await.unwrap();
+        drop(store);
+    }
+}

From f9548e6358a43603f45b9c0d15e0499e2db87f5b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 6 Jul 2023 14:27:57 +0800
Subject: [PATCH 037/261] feat: introduce prometheus metrics (#54)

* feat: introduce prometheus metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* calc metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix registry metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |  1 +
 foyer-storage/src/flusher.rs    | 10 ++++
 foyer-storage/src/lib.rs        |  1 +
 foyer-storage/src/metrics.rs    | 94 +++++++++++++++++++++++++++++++++
 foyer-storage/src/reclaimer.rs  | 11 ++++
 foyer-storage/src/store.rs      | 49 +++++++++++++++--
 6 files changed, 162 insertions(+), 4 deletions(-)
 create mode 100644 foyer-storage/src/metrics.rs

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 78cea50f..b2d85e83 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -214,6 +214,7 @@ async fn main() {
         flushers: args.flushers,
         reclaimers: args.reclaimers,
         recover_concurrency: args.recover_concurrency,
+        prometheus_registry: None,
     };
 
     println!("{:#?}", config);
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 7f3f96a1..6279d274 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -25,6 +25,7 @@ use tokio::{
 use crate::{
     device::{BufferAllocator, Device},
     error::{Error, Result},
+    metrics::Metrics,
     region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
     slice::Slice,
@@ -64,6 +65,7 @@ impl Flusher {
         buffers: Arc<AsyncQueue<Vec<u8, A>>>,
         region_manager: Arc<RegionManager<A, D, E, EL>>,
         stop_rxs: Vec<broadcast::Receiver<()>>,
+        metrics: Arc<Metrics>,
     ) -> Vec<JoinHandle<()>>
     where
         A: BufferAllocator,
@@ -88,6 +90,7 @@ impl Flusher {
                 buffers: buffers.clone(),
                 region_manager: region_manager.clone(),
                 stop_rx,
+                metrics: metrics.clone(),
             })
             .collect_vec();
 
@@ -126,6 +129,8 @@ where
     region_manager: Arc<RegionManager<A, D, E, EL>>,
 
     stop_rx: broadcast::Receiver<()>,
+
+    metrics: Arc<Metrics>,
 }
 
 impl<A, D, E, EL> Runner<A, D, E, EL>
@@ -211,6 +216,11 @@ where
 
         tracing::info!("[flusher] finish flush task, region: {}", task.region_id);
 
+        self.metrics
+            .bytes_flush
+            .inc_by(region.device().region_size() as u64);
+        self.metrics.size.add(region.device().region_size() as i64);
+
         Ok(())
     }
 }
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index ccda3835..a0106337 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -26,6 +26,7 @@ pub mod device;
 pub mod error;
 pub mod flusher;
 pub mod indices;
+pub mod metrics;
 pub mod reclaimer;
 pub mod region;
 pub mod region_manager;
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
new file mode 100644
index 00000000..0bdea154
--- /dev/null
+++ b/foyer-storage/src/metrics.rs
@@ -0,0 +1,94 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use prometheus::{
+    register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
+    register_int_gauge_with_registry, Histogram, IntCounter, IntGauge, Registry,
+};
+
+#[derive(Debug)]
+pub struct Metrics {
+    pub latency_insert: Histogram,
+    pub latency_lookup_hit: Histogram,
+    pub latency_lookup_miss: Histogram,
+    pub latency_remove: Histogram,
+
+    pub bytes_insert: IntCounter,
+    pub bytes_lookup: IntCounter,
+    pub bytes_flush: IntCounter,
+    pub bytes_reclaim: IntCounter,
+    pub bytes_reinsert: IntCounter,
+
+    pub size: IntGauge,
+}
+
+impl Default for Metrics {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Metrics {
+    pub fn new() -> Self {
+        Self::with_registry(Registry::default())
+    }
+
+    pub fn with_registry(registry: Registry) -> Self {
+        let latency = register_histogram_vec_with_registry!(
+            "foyer_storage_latency",
+            "foyer storage latency",
+            &["op", "extra"],
+            vec![0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
+            registry
+        )
+        .unwrap();
+        let bytes = register_int_counter_vec_with_registry!(
+            "foyer_storage_bytes",
+            "foyer storage bytes",
+            &["op", "extra"],
+            registry
+        )
+        .unwrap();
+
+        let latency_insert = latency.with_label_values(&["insert", ""]);
+        let latency_lookup_hit = latency.with_label_values(&["lookup", "hit"]);
+        let latency_lookup_miss = latency.with_label_values(&["lookup", "miss"]);
+        let latency_remove = latency.with_label_values(&["remove", ""]);
+
+        let bytes_insert = bytes.with_label_values(&["insert", ""]);
+        let bytes_lookup = bytes.with_label_values(&["lookup", ""]);
+        let bytes_flush = bytes.with_label_values(&["flush", ""]);
+        let bytes_reclaim = bytes.with_label_values(&["reclaim", ""]);
+        let bytes_reinsert = bytes.with_label_values(&["reinsert", ""]);
+
+        let size =
+            register_int_gauge_with_registry!("foyer_storage_size", "foyer storage size", registry)
+                .unwrap();
+
+        Self {
+            latency_insert,
+            latency_lookup_hit,
+            latency_lookup_miss,
+            latency_remove,
+
+            bytes_insert,
+            bytes_lookup,
+            bytes_flush,
+            bytes_reclaim,
+            bytes_reinsert,
+
+            size,
+        }
+    }
+}
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 73c4af9b..a18bbb63 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -19,6 +19,7 @@ use crate::{
     device::{BufferAllocator, Device},
     error::{Error, Result},
     indices::Indices,
+    metrics::Metrics,
     region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
@@ -66,6 +67,7 @@ impl Reclaimer {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub async fn run<K, V, A, D, EP, AP, RP, EL>(
         &self,
         store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
@@ -74,6 +76,7 @@ impl Reclaimer {
         reinsertion: RP,
         indices: Arc<Indices<K>>,
         stop_rxs: Vec<broadcast::Receiver<()>>,
+        metrics: Arc<Metrics>,
     ) -> Vec<JoinHandle<()>>
     where
         K: Key,
@@ -105,6 +108,7 @@ impl Reclaimer {
                 _reinsertion: reinsertion.clone(),
                 indices: indices.clone(),
                 stop_rx,
+                metrics: metrics.clone(),
             })
             .collect_vec();
 
@@ -153,6 +157,8 @@ where
     indices: Arc<Indices<K>>,
 
     stop_rx: broadcast::Receiver<()>,
+
+    metrics: Arc<Metrics>,
 }
 
 impl<K, V, A, D, EP, AP, RP, EL> Runner<K, V, A, D, EP, AP, RP, EL>
@@ -226,6 +232,11 @@ where
             task.region_id
         );
 
+        self.metrics
+            .bytes_reclaim
+            .inc_by(region.device().region_size() as u64);
+        self.metrics.size.sub(region.device().region_size() as i64);
+
         Ok(())
     }
 }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 358d9c11..29c5edc8 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc};
+use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant};
 
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, queue::AsyncQueue};
@@ -28,6 +28,7 @@ use crate::{
     error::{Error, Result},
     flusher::Flusher,
     indices::{Index, Indices},
+    metrics::Metrics,
     reclaimer::Reclaimer,
     region::{Region, RegionId},
     region_manager::{RegionEpItemAdapter, RegionManager},
@@ -54,6 +55,7 @@ where
     pub flushers: usize,
     pub reclaimers: usize,
     pub recover_concurrency: usize,
+    pub prometheus_registry: Option<prometheus::Registry>,
 }
 
 impl<D, AP, EP, RP, EL> Debug for StoreConfig<D, AP, EP, RP, EL>
@@ -100,6 +102,8 @@ where
 
     stop_tx: broadcast::Sender<()>,
 
+    metrics: Arc<Metrics>,
+
     _marker: PhantomData<(V, RP)>,
 }
 
@@ -151,6 +155,11 @@ where
             .map(|_| stop_tx.subscribe())
             .collect_vec();
 
+        let metrics = match config.prometheus_registry {
+            Some(registry) => Arc::new(crate::metrics::Metrics::with_registry(registry)),
+            None => Arc::new(crate::metrics::Metrics::new()),
+        };
+
         let store = Arc::new(Self {
             indices: indices.clone(),
             region_manager: region_manager.clone(),
@@ -158,6 +167,7 @@ where
             admission: config.admission,
             handles: Mutex::new(vec![]),
             stop_tx,
+            metrics: metrics.clone(),
             _marker: PhantomData,
         });
 
@@ -166,7 +176,12 @@ where
         let mut handles = vec![];
         handles.append(
             &mut flusher
-                .run(buffers, region_manager.clone(), flusher_stop_rxs)
+                .run(
+                    buffers,
+                    region_manager.clone(),
+                    flusher_stop_rxs,
+                    metrics.clone(),
+                )
                 .await,
         );
         handles.append(
@@ -178,6 +193,7 @@ where
                     config.reinsertion,
                     indices,
                     reclaimer_stop_rxs,
+                    metrics,
                 )
                 .await,
         );
@@ -197,11 +213,14 @@ where
     }
 
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
+        let _timer = self.metrics.latency_insert.start_timer();
+
         if !self.admission.judge(&key, &value) {
             return Ok(false);
         }
 
         let serialized_len = self.serialized_len(&key, &value);
+        self.metrics.bytes_insert.inc_by(serialized_len as u64);
 
         let mut slice = match self.region_manager.allocate(serialized_len).await {
             crate::region::AllocateResult::Ok(slice) => slice,
@@ -238,9 +257,16 @@ where
     }
 
     pub async fn lookup(&self, key: &K) -> Result<Option<V>> {
+        let now = Instant::now();
+
         let index = match self.indices.lookup(key) {
             Some(index) => index,
-            None => return Ok(None),
+            None => {
+                self.metrics
+                    .latency_lookup_miss
+                    .observe(now.elapsed().as_secs_f64());
+                return Ok(None);
+            }
         };
 
         self.region_manager.record_access(&index.region);
@@ -251,18 +277,31 @@ where
         // TODO(MrCroxx): read value only
         let slice = match region.load(start..end, index.version).await? {
             Some(slice) => slice,
-            None => return Ok(None),
+            None => {
+                self.metrics
+                    .latency_lookup_miss
+                    .observe(now.elapsed().as_secs_f64());
+                return Ok(None);
+            }
         };
+        self.metrics.bytes_lookup.inc_by(slice.len() as u64);
 
         let res = match read_entry::<K, V>(slice.as_ref()) {
             Some((_key, value)) => Ok(Some(value)),
             None => Ok(None),
         };
         slice.destroy().await;
+
+        self.metrics
+            .latency_lookup_hit
+            .observe(now.elapsed().as_secs_f64());
+
         res
     }
 
     pub fn remove(&self, key: &K) {
+        let _timer = self.metrics.latency_remove.start_timer();
+
         self.indices.remove(key);
     }
 
@@ -656,6 +695,7 @@ pub mod tests {
             flushers: 1,
             reclaimers: 1,
             recover_concurrency: 2,
+            prometheus_registry: None,
         };
 
         let store = TestStore::open(config).await.unwrap();
@@ -699,6 +739,7 @@ pub mod tests {
             flushers: 1,
             reclaimers: 1,
             recover_concurrency: 2,
+            prometheus_registry: None,
         };
         let store = TestStore::open(config).await.unwrap();
 

From 3c0a0ca9b93f134ff96a2fe1f6946e4f2d4d22d8 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 6 Jul 2023 14:41:40 +0800
Subject: [PATCH 038/261] fix: recovery test (#55)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 29c5edc8..048fbbc4 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -737,7 +737,7 @@ pub mod tests {
             reinsertion: ReinsertNone::default(),
             buffer_pool_size: 8 * MB,
             flushers: 1,
-            reclaimers: 1,
+            reclaimers: 0,
             recover_concurrency: 2,
             prometheus_registry: None,
         };

From 809ca83f755dc37889451c45e8e12af1483704ab Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 6 Jul 2023 17:27:19 +0800
Subject: [PATCH 039/261] feat: support multiple admission or reinsertion
 policies, loose lock (#56)

- support multiple admission or reinsertion policies
- loose flush exclusive lock

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs  |  6 +--
 foyer-storage/src/flusher.rs     | 36 +++++++++---------
 foyer-storage/src/lib.rs         | 30 ++++++---------
 foyer-storage/src/reclaimer.rs   | 23 ++++-------
 foyer-storage/src/reinsertion.rs |  2 +-
 foyer-storage/src/store.rs       | 65 +++++++++++++-------------------
 6 files changed, 69 insertions(+), 93 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index b2d85e83..20639646 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -121,7 +121,7 @@ impl Args {
     }
 }
 
-type TStore = LfuFsStore<u64, Vec<u8>, AdmitAll<u64, Vec<u8>>, ReinsertNone<u64, Vec<u8>>>;
+type TStore = LfuFsStore<u64, Vec<u8>>;
 
 fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
@@ -208,8 +208,8 @@ async fn main() {
     let config = StoreConfig {
         eviction_config,
         device_config,
-        admission: AdmitAll::default(),
-        reinsertion: ReinsertNone::default(),
+        admissions: vec![Arc::new(AdmitAll::default())],
+        reinsertions: vec![Arc::new(ReinsertNone::default())],
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
         flushers: args.flushers,
         reclaimers: args.reclaimers,
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 6279d274..e5145741 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -162,31 +162,31 @@ where
 
         tracing::trace!("[flusher] step 1");
 
-        {
-            // step 1: write buffer back to device
-            let slice = region.load(.., 0).await?.unwrap();
+        // step 1: write buffer back to device
+        let slice = region.load(.., 0).await?.unwrap();
 
+        {
             // wait all physical readers (from previous version) and writers done
             let guard = region.exclusive(false, true, false).await;
+            drop(guard);
+        }
 
-            tracing::trace!("[flusher] write region {} back to device", task.region_id);
+        tracing::trace!("[flusher] write region {} back to device", task.region_id);
 
-            let mut offset = 0;
-            let len = region.device().io_size();
-            while offset < region.device().region_size() {
-                let start = offset;
-                let end = std::cmp::min(offset + len, region.device().region_size());
+        let mut offset = 0;
+        let len = region.device().io_size();
+        while offset < region.device().region_size() {
+            let start = offset;
+            let end = std::cmp::min(offset + len, region.device().region_size());
 
-                let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
-                region
-                    .device()
-                    .write(s, region.id(), offset as u64, len)
-                    .await?;
-                offset += len;
-            }
-            drop(guard);
-            slice.destroy().await;
+            let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
+            region
+                .device()
+                .write(s, region.id(), offset as u64, len)
+                .await?;
+            offset += len;
         }
+        slice.destroy().await;
 
         tracing::trace!("[flusher] step 2");
 
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index a0106337..e758c6fb 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -34,7 +34,7 @@ pub mod reinsertion;
 pub mod slice;
 pub mod store;
 
-pub type LruFsStore<K, V, AP, RP> = store::Store<
+pub type LruFsStore<K, V> = store::Store<
     K,
     V,
     AlignedAllocator,
@@ -42,22 +42,20 @@ pub type LruFsStore<K, V, AP, RP> = store::Store<
     foyer_intrusive::eviction::lru::Lru<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
     >,
-    AP,
-    RP,
     foyer_intrusive::eviction::lru::LruLink,
 >;
 
-pub type LruFsStoreConfig<AP, RP> = store::StoreConfig<
+pub type LruFsStoreConfig<K, V> = store::StoreConfig<
+    K,
+    V,
     device::fs::FsDevice,
-    AP,
     foyer_intrusive::eviction::lru::Lru<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
     >,
-    RP,
     foyer_intrusive::eviction::lru::LruLink,
 >;
 
-pub type LfuFsStore<K, V, AP, RP> = store::Store<
+pub type LfuFsStore<K, V> = store::Store<
     K,
     V,
     AlignedAllocator,
@@ -65,22 +63,20 @@ pub type LfuFsStore<K, V, AP, RP> = store::Store<
     foyer_intrusive::eviction::lfu::Lfu<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
     >,
-    AP,
-    RP,
     foyer_intrusive::eviction::lfu::LfuLink,
 >;
 
-pub type LfuFsStoreConfig<AP, RP> = store::StoreConfig<
+pub type LfuFsStoreConfig<K, V> = store::StoreConfig<
+    K,
+    V,
     device::fs::FsDevice,
-    AP,
     foyer_intrusive::eviction::lfu::Lfu<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
     >,
-    RP,
     foyer_intrusive::eviction::lfu::LfuLink,
 >;
 
-pub type FifoFsStore<K, V, AP, RP> = store::Store<
+pub type FifoFsStore<K, V> = store::Store<
     K,
     V,
     AlignedAllocator,
@@ -88,17 +84,15 @@ pub type FifoFsStore<K, V, AP, RP> = store::Store<
     foyer_intrusive::eviction::fifo::Fifo<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
     >,
-    AP,
-    RP,
     foyer_intrusive::eviction::fifo::FifoLink,
 >;
 
-pub type FifoFsStoreConfig<AP, RP> = store::StoreConfig<
+pub type FifoFsStoreConfig<K, V> = store::StoreConfig<
+    K,
+    V,
     device::fs::FsDevice,
-    AP,
     foyer_intrusive::eviction::fifo::Fifo<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
     >,
-    RP,
     foyer_intrusive::eviction::fifo::FifoLink,
 >;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index a18bbb63..71318eec 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -15,7 +15,6 @@
 use std::sync::Arc;
 
 use crate::{
-    admission::AdmissionPolicy,
     device::{BufferAllocator, Device},
     error::{Error, Result},
     indices::Indices,
@@ -68,12 +67,12 @@ impl Reclaimer {
     }
 
     #[allow(clippy::too_many_arguments)]
-    pub async fn run<K, V, A, D, EP, AP, RP, EL>(
+    pub async fn run<K, V, A, D, EP, EL>(
         &self,
-        store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
+        store: Arc<Store<K, V, A, D, EP, EL>>,
         region_manager: Arc<RegionManager<A, D, EP, EL>>,
         clean_regions: Arc<AsyncQueue<RegionId>>,
-        reinsertion: RP,
+        reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
         indices: Arc<Indices<K>>,
         stop_rxs: Vec<broadcast::Receiver<()>>,
         metrics: Arc<Metrics>,
@@ -84,8 +83,6 @@ impl Reclaimer {
         A: BufferAllocator,
         D: Device<IoBufferAllocator = A>,
         EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-        AP: AdmissionPolicy<Key = K, Value = V>,
-        RP: ReinsertionPolicy<Key = K, Value = V>,
         EL: Link,
     {
         let mut inner = self.inner.lock().await;
@@ -105,7 +102,7 @@ impl Reclaimer {
                 _store: store.clone(),
                 region_manager: region_manager.clone(),
                 clean_regions: clean_regions.clone(),
-                _reinsertion: reinsertion.clone(),
+                _reinsertions: reinsertions.clone(),
                 indices: indices.clone(),
                 stop_rx,
                 metrics: metrics.clone(),
@@ -137,23 +134,21 @@ impl Reclaimer {
     }
 }
 
-struct Runner<K, V, A, D, EP, AP, RP, EL>
+struct Runner<K, V, A, D, EP, EL>
 where
     K: Key,
     V: Value,
     A: BufferAllocator,
     D: Device<IoBufferAllocator = A>,
     EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    AP: AdmissionPolicy<Key = K, Value = V>,
-    RP: ReinsertionPolicy<Key = K, Value = V>,
     EL: Link,
 {
     task_rx: mpsc::Receiver<ReclaimTask>,
 
-    _store: Arc<Store<K, V, A, D, EP, AP, RP, EL>>,
+    _store: Arc<Store<K, V, A, D, EP, EL>>,
     region_manager: Arc<RegionManager<A, D, EP, EL>>,
     clean_regions: Arc<AsyncQueue<RegionId>>,
-    _reinsertion: RP,
+    _reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
     indices: Arc<Indices<K>>,
 
     stop_rx: broadcast::Receiver<()>,
@@ -161,15 +156,13 @@ where
     metrics: Arc<Metrics>,
 }
 
-impl<K, V, A, D, EP, AP, RP, EL> Runner<K, V, A, D, EP, AP, RP, EL>
+impl<K, V, A, D, EP, EL> Runner<K, V, A, D, EP, EL>
 where
     K: Key,
     V: Value,
     A: BufferAllocator,
     D: Device<IoBufferAllocator = A>,
     EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    AP: AdmissionPolicy<Key = K, Value = V>,
-    RP: ReinsertionPolicy<Key = K, Value = V>,
     EL: Link,
 {
     async fn run(mut self) -> Result<()> {
diff --git a/foyer-storage/src/reinsertion.rs b/foyer-storage/src/reinsertion.rs
index 9928a86f..b1e59e10 100644
--- a/foyer-storage/src/reinsertion.rs
+++ b/foyer-storage/src/reinsertion.rs
@@ -18,7 +18,7 @@ use foyer_common::code::{Key, Value};
 
 use std::fmt::Debug;
 
-pub trait ReinsertionPolicy: Send + Sync + 'static + Clone + Debug {
+pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 048fbbc4..7ffd97db 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -39,18 +39,18 @@ use std::hash::Hasher;
 
 const REGION_MAGIC: u64 = 0x19970327;
 
-pub struct StoreConfig<D, AP, EP, RP, EL>
+pub struct StoreConfig<K, V, D, EP, EL>
 where
+    K: Key,
+    V: Value,
     D: Device,
-    AP: AdmissionPolicy,
     EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    RP: ReinsertionPolicy,
     EL: Link,
 {
     pub eviction_config: EP::Config,
     pub device_config: D::Config,
-    pub admission: AP,
-    pub reinsertion: RP,
+    pub admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
+    pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
     pub buffer_pool_size: usize,
     pub flushers: usize,
     pub reclaimers: usize,
@@ -58,20 +58,20 @@ where
     pub prometheus_registry: Option<prometheus::Registry>,
 }
 
-impl<D, AP, EP, RP, EL> Debug for StoreConfig<D, AP, EP, RP, EL>
+impl<K, V, D, EP, EL> Debug for StoreConfig<K, V, D, EP, EL>
 where
+    K: Key,
+    V: Value,
     D: Device,
-    AP: AdmissionPolicy,
     EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    RP: ReinsertionPolicy,
     EL: Link,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("StoreConfig")
             .field("eviction_config", &self.eviction_config)
             .field("device_config", &self.device_config)
-            .field("admission", &self.admission)
-            .field("reinsertion", &self.reinsertion)
+            .field("admissions", &self.admissions)
+            .field("reinsertions", &self.reinsertions)
             .field("buffer_pool_size", &self.buffer_pool_size)
             .field("flushers", &self.flushers)
             .field("reclaimers", &self.reclaimers)
@@ -79,15 +79,13 @@ where
     }
 }
 
-pub struct Store<K, V, BA, D, EP, AP, RP, EL>
+pub struct Store<K, V, BA, D, EP, EL>
 where
     K: Key,
     V: Value,
     BA: BufferAllocator,
     D: Device<IoBufferAllocator = BA>,
     EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    AP: AdmissionPolicy<Key = K, Value = V>,
-    RP: ReinsertionPolicy<Key = K, Value = V>,
     EL: Link,
 {
     indices: Arc<Indices<K>>,
@@ -96,7 +94,7 @@ where
 
     device: D,
 
-    admission: AP,
+    admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
 
     handles: Mutex<Vec<JoinHandle<()>>>,
 
@@ -104,21 +102,19 @@ where
 
     metrics: Arc<Metrics>,
 
-    _marker: PhantomData<(V, RP)>,
+    _marker: PhantomData<V>,
 }
 
-impl<K, V, BA, D, EP, AP, RP, EL> Store<K, V, BA, D, EP, AP, RP, EL>
+impl<K, V, BA, D, EP, EL> Store<K, V, BA, D, EP, EL>
 where
     K: Key,
     V: Value,
     BA: BufferAllocator,
     D: Device<IoBufferAllocator = BA>,
     EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    AP: AdmissionPolicy<Key = K, Value = V>,
-    RP: ReinsertionPolicy<Key = K, Value = V>,
     EL: Link,
 {
-    pub async fn open(config: StoreConfig<D, AP, EP, RP, EL>) -> Result<Arc<Self>> {
+    pub async fn open(config: StoreConfig<K, V, D, EP, EL>) -> Result<Arc<Self>> {
         tracing::info!("open store with config:\n{:#?}", config);
 
         let device = D::open(config.device_config).await?;
@@ -164,7 +160,7 @@ where
             indices: indices.clone(),
             region_manager: region_manager.clone(),
             device: device.clone(),
-            admission: config.admission,
+            admissions: config.admissions,
             handles: Mutex::new(vec![]),
             stop_tx,
             metrics: metrics.clone(),
@@ -190,7 +186,7 @@ where
                     store.clone(),
                     region_manager,
                     clean_regions,
-                    config.reinsertion,
+                    config.reinsertions,
                     indices,
                     reclaimer_stop_rxs,
                     metrics,
@@ -215,8 +211,10 @@ where
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
         let _timer = self.metrics.latency_insert.start_timer();
 
-        if !self.admission.judge(&key, &value) {
-            return Ok(false);
+        for admission in &self.admissions {
+            if !admission.judge(&key, &value) {
+                return Ok(false);
+            }
         }
 
         let serialized_len = self.serialized_len(&key, &value);
@@ -318,7 +316,6 @@ where
             .await
         {
             crate::region::AllocateResult::Full { mut slice, remain } => {
-                println!("seal");
                 // current region is full, write region footer and try allocate again
                 let footer = RegionFooter {
                     magic: REGION_MAGIC,
@@ -328,7 +325,6 @@ where
                 slice.destroy().await;
             }
             crate::region::AllocateResult::Ok(slice) => {
-                println!("not seal");
                 // region is empty, skip
                 slice.destroy().await
             }
@@ -657,18 +653,11 @@ pub mod tests {
         AlignedAllocator,
         FsDevice,
         Fifo<RegionEpItemAdapter<FifoLink>>,
-        AdmitAll<u64, Vec<u8>>,
-        ReinsertNone<u64, Vec<u8>>,
         FifoLink,
     >;
 
-    type TestStoreConfig = StoreConfig<
-        FsDevice,
-        AdmitAll<u64, Vec<u8>>,
-        Fifo<RegionEpItemAdapter<FifoLink>>,
-        ReinsertNone<u64, Vec<u8>>,
-        FifoLink,
-    >;
+    type TestStoreConfig =
+        StoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
     #[tokio::test]
     #[allow(clippy::identity_op)]
@@ -689,8 +678,8 @@ pub mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            admission: AdmitAll::default(),
-            reinsertion: ReinsertNone::default(),
+            admissions: vec![Arc::new(AdmitAll::default())],
+            reinsertions: vec![Arc::new(ReinsertNone::default())],
             buffer_pool_size: 8 * MB,
             flushers: 1,
             reclaimers: 1,
@@ -733,8 +722,8 @@ pub mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            admission: AdmitAll::default(),
-            reinsertion: ReinsertNone::default(),
+            admissions: vec![Arc::new(AdmitAll::default())],
+            reinsertions: vec![Arc::new(ReinsertNone::default())],
             buffer_pool_size: 8 * MB,
             flushers: 1,
             reclaimers: 0,

From 32ec8805d269070701e20c4bdfb4e4b8d5aac0b0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 7 Jul 2023 16:09:56 +0800
Subject: [PATCH 040/261] feat: introduce rated random admission policy (#57)

* feat: introduce rated random admission policy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* make clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml                 |   9 +-
 .github/workflows/main.yml                    |   7 +-
 .github/workflows/pull-request.yml            |   7 +-
 foyer-common/src/code.rs                      |  10 +-
 foyer-intrusive/src/core/adapter.rs           |  14 ++
 foyer-storage/src/admission/all.rs            |  38 ++++
 .../src/{admission.rs => admission/mod.rs}    |   6 +
 foyer-storage/src/admission/rated_random.rs   | 186 ++++++++++++++++++
 foyer-storage/src/store.rs                    |   3 +
 9 files changed, 269 insertions(+), 11 deletions(-)
 create mode 100644 foyer-storage/src/admission/all.rs
 rename foyer-storage/src/{admission.rs => admission/mod.rs} (91%)
 create mode 100644 foyer-storage/src/admission/rated_random.rs

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 912afa7b..c96b3a3d 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -66,6 +66,9 @@ jobs:
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@nextest
+      - name: Run rust test with coverage (igored tests)
+        run: |
+          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests::test_rated_random --exact --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
@@ -94,7 +97,7 @@ jobs:
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
-          RUSTFLAGS: '--cfg tokio_unstable'
+          RUSTFLAGS: "--cfg tokio_unstable"
           RUST_LOG: info
           TOKIO_WORKER_THREADS: 1
         run: |-
@@ -125,9 +128,9 @@ jobs:
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
-          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
\ No newline at end of file
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4dd59d3a..926eeeee 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -73,6 +73,9 @@ jobs:
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@nextest
+      - name: Run rust test with coverage (igored tests)
+        run: |
+          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests::test_rated_random --exact --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
@@ -100,7 +103,7 @@ jobs:
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
-          RUSTFLAGS: '--cfg tokio_unstable'
+          RUSTFLAGS: "--cfg tokio_unstable"
           RUST_LOG: info
           TOKIO_WORKER_THREADS: 1
         run: |-
@@ -130,7 +133,7 @@ jobs:
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
-          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index c949d49e..eea70e28 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -72,6 +72,9 @@ jobs:
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@nextest
+      - name: Run rust test with coverage (igored tests)
+        run: |
+          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests::test_rated_random --exact --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
@@ -99,7 +102,7 @@ jobs:
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
-          RUSTFLAGS: '--cfg tokio_unstable'
+          RUSTFLAGS: "--cfg tokio_unstable"
           RUST_LOG: info
           TOKIO_WORKER_THREADS: 1
         run: |-
@@ -129,7 +132,7 @@ jobs:
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
-          RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 0c34d919..5cd298b2 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -15,6 +15,7 @@
 use bytes::{Buf, BufMut};
 use paste::paste;
 
+#[allow(unused)]
 pub trait Key:
     Sized
     + Send
@@ -32,25 +33,26 @@ pub trait Key:
         panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
     }
 
-    fn write(&self, _buf: &mut [u8]) {
+    fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Key` if storage is used.")
     }
 
-    fn read(_buf: &[u8]) -> Self {
+    fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
 }
 
+#[allow(unused)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
     }
 
-    fn write(&self, _buf: &mut [u8]) {
+    fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Value` if storage is used.")
     }
 
-    fn read(_buf: &[u8]) -> Self {
+    fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
 }
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index c41b1bf0..6f5778b6 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -12,6 +12,20 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+//  Copyright 2020 Amari Robinson
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
 use std::fmt::Debug;
 
 use foyer_common::code::Key;
diff --git a/foyer-storage/src/admission/all.rs b/foyer-storage/src/admission/all.rs
new file mode 100644
index 00000000..78814fe0
--- /dev/null
+++ b/foyer-storage/src/admission/all.rs
@@ -0,0 +1,38 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::marker::PhantomData;
+
+use foyer_common::code::{Key, Value};
+
+use super::AdmissionPolicy;
+
+#[derive(Debug)]
+pub struct AdmitAll<K: Key, V: Value>(PhantomData<(K, V)>);
+
+impl<K: Key, V: Value> Default for AdmitAll<K, V> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<K: Key, V: Value> AdmissionPolicy for AdmitAll<K, V> {
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, _key: &Self::Key, _value: &Self::Value) -> bool {
+        true
+    }
+}
diff --git a/foyer-storage/src/admission.rs b/foyer-storage/src/admission/mod.rs
similarity index 91%
rename from foyer-storage/src/admission.rs
rename to foyer-storage/src/admission/mod.rs
index b953a4f7..ad3921d1 100644
--- a/foyer-storage/src/admission.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -18,11 +18,14 @@ use foyer_common::code::{Key, Value};
 
 use std::fmt::Debug;
 
+#[allow(unused)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
     fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
+
+    fn admit(&self, key: &Self::Key, value: &Self::Value) {}
 }
 
 #[derive(Debug)]
@@ -43,3 +46,6 @@ impl<K: Key, V: Value> AdmissionPolicy for AdmitAll<K, V> {
         true
     }
 }
+
+pub mod all;
+pub mod rated_random;
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
new file mode 100644
index 00000000..539d24cd
--- /dev/null
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -0,0 +1,186 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    sync::atomic::{AtomicUsize, Ordering},
+    time::{Duration, Instant},
+};
+
+use foyer_common::code::{Key, Value};
+use parking_lot::{Mutex, MutexGuard};
+use rand::{thread_rng, Rng};
+
+use super::AdmissionPolicy;
+
+const PRECISION: usize = 100000;
+
+/// p       : admit probability among △t
+/// w(t)    : weight at time t
+/// r       : actual admitted rate
+/// E(w)    : expected admitted weight
+/// E(r)    : expceted admitted rate
+///
+/// E(w(t)) = p * w(t)
+/// r = sum(△t){ admitted w(t) } / △t
+/// E(r) = sum(△t){ E(w(t)) } / △t
+///      = sum(△t){ p * w(t) } / △t
+///      = p / △t * sum(△t){ w(t) }
+/// p = E(r) * △t / sum(△t){ w(t) }
+pub struct RatedRandom<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    rate: usize,
+    update_interval: Duration,
+
+    bytes: AtomicUsize,
+    probability: AtomicUsize,
+
+    inner: Mutex<Inner<K, V>>,
+}
+
+impl<K, V> Debug for RatedRandom<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DynamicRandom").finish()
+    }
+}
+
+struct Inner<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    last_update_time: Option<Instant>,
+    last_bytes: usize,
+
+    _marker: PhantomData<(K, V)>,
+}
+
+impl<K, V> RatedRandom<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn new(rate: usize, update_interval: Duration) -> Self {
+        Self {
+            rate,
+            update_interval,
+            bytes: AtomicUsize::new(0),
+            probability: AtomicUsize::new(0),
+            inner: Mutex::new(Inner {
+                last_update_time: None,
+                last_bytes: 0,
+                _marker: PhantomData,
+            }),
+        }
+    }
+
+    fn judge(&self, key: &K, value: &V) -> bool {
+        if let Some(inner) = self.inner.try_lock() {
+            self.update(inner);
+        }
+
+        // TODO(MrCroxx): unify weighter?
+        let weight = key.serialized_len() + value.serialized_len();
+        self.bytes.fetch_add(weight, Ordering::Relaxed);
+
+        thread_rng().gen_range(0..PRECISION) < self.probability.load(Ordering::Relaxed)
+    }
+
+    fn update(&self, mut inner: MutexGuard<'_, Inner<K, V>>) {
+        let now = Instant::now();
+
+        let elapsed = match inner.last_update_time {
+            Some(last_update_time) => now.duration_since(last_update_time),
+            None => self.update_interval,
+        };
+
+        if elapsed < self.update_interval {
+            return;
+        }
+
+        let bytes = self.bytes.load(Ordering::Relaxed);
+        let bytes_delta = std::cmp::max(bytes - inner.last_bytes, 1);
+
+        let p = self.rate as f64 * elapsed.as_secs_f64() / bytes_delta as f64;
+        let p = (p * PRECISION as f64) as usize;
+        self.probability.store(p, Ordering::Relaxed);
+
+        inner.last_update_time = Some(now);
+        inner.last_bytes = bytes;
+    }
+}
+
+impl<K, V> AdmissionPolicy for RatedRandom<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool {
+        self.judge(key, value)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use super::*;
+
+    #[ignore]
+    #[tokio::test]
+    async fn test_rated_random() {
+        const RATE: usize = 1_000_000;
+        const ERATIO: f64 = 0.1;
+
+        let score = Arc::new(AtomicUsize::new(0));
+
+        let rr = Arc::new(RatedRandom::<u64, Vec<u8>>::new(
+            RATE,
+            Duration::from_millis(100),
+        ));
+
+        async fn submit(rr: Arc<RatedRandom<u64, Vec<u8>>>, score: Arc<AtomicUsize>) {
+            loop {
+                tokio::time::sleep(Duration::from_millis(10)).await;
+                let size = thread_rng().gen_range(1000..10000);
+                if rr.judge(&0, &vec![0; size]) {
+                    score.fetch_add(size, Ordering::Relaxed);
+                }
+            }
+        }
+
+        for _ in 0..10 {
+            tokio::spawn(submit(rr.clone(), score.clone()));
+        }
+
+        tokio::time::sleep(Duration::from_secs(10)).await;
+        let s = score.load(Ordering::Relaxed);
+        let error = (s as isize - RATE as isize * 10).unsigned_abs();
+        let eratio = error as f64 / (RATE as f64 * 10.0);
+
+        assert!(eratio < ERATIO);
+    }
+}
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 7ffd97db..a5fffb0d 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -216,6 +216,9 @@ where
                 return Ok(false);
             }
         }
+        for admission in &self.admissions {
+            admission.admit(&key, &value);
+        }
 
         let serialized_len = self.serialized_len(&key, &value);
         self.metrics.bytes_insert.inc_by(serialized_len as u64);

From d037322438164c3a316792218c04dfa94a9dc04e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 7 Jul 2023 16:29:42 +0800
Subject: [PATCH 041/261] feat: add rated random option for bench args (#58)

---
 foyer-storage-bench/src/main.rs    | 19 ++++++++++++---
 foyer-storage/src/admission/all.rs | 38 ------------------------------
 foyer-storage/src/admission/mod.rs |  1 -
 foyer-storage/src/reinsertion.rs   | 27 ---------------------
 foyer-storage/src/store.rs         | 18 ++++++--------
 5 files changed, 23 insertions(+), 80 deletions(-)
 delete mode 100644 foyer-storage/src/admission/all.rs

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 20639646..51fe9bb1 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -33,7 +33,9 @@ use clap::Parser;
 
 use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
-    admission::AdmitAll, device::fs::FsDeviceConfig, reinsertion::ReinsertNone, store::StoreConfig,
+    admission::{rated_random::RatedRandom, AdmissionPolicy},
+    device::fs::FsDeviceConfig,
+    store::StoreConfig,
     LfuFsStore,
 };
 use futures::future::join_all;
@@ -113,6 +115,11 @@ pub struct Args {
 
     #[arg(long, default_value_t = 16)]
     recover_concurrency: usize,
+
+    /// enable rated random admission policy if `rated_random` > 0
+    /// (MiB)
+    #[arg(long, default_value_t = 0)]
+    rated_random: usize,
 }
 
 impl Args {
@@ -205,11 +212,17 @@ async fn main() {
         io_size: args.io_size,
     };
 
+    let mut admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
+    if args.rated_random > 0 {
+        let rr = RatedRandom::new(args.rated_random * 1024 * 1024, Duration::from_millis(100));
+        admissions.push(Arc::new(rr));
+    }
+
     let config = StoreConfig {
         eviction_config,
         device_config,
-        admissions: vec![Arc::new(AdmitAll::default())],
-        reinsertions: vec![Arc::new(ReinsertNone::default())],
+        admissions,
+        reinsertions: vec![],
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
         flushers: args.flushers,
         reclaimers: args.reclaimers,
diff --git a/foyer-storage/src/admission/all.rs b/foyer-storage/src/admission/all.rs
deleted file mode 100644
index 78814fe0..00000000
--- a/foyer-storage/src/admission/all.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::marker::PhantomData;
-
-use foyer_common::code::{Key, Value};
-
-use super::AdmissionPolicy;
-
-#[derive(Debug)]
-pub struct AdmitAll<K: Key, V: Value>(PhantomData<(K, V)>);
-
-impl<K: Key, V: Value> Default for AdmitAll<K, V> {
-    fn default() -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<K: Key, V: Value> AdmissionPolicy for AdmitAll<K, V> {
-    type Key = K;
-
-    type Value = V;
-
-    fn judge(&self, _key: &Self::Key, _value: &Self::Value) -> bool {
-        true
-    }
-}
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index ad3921d1..9b7ad6fe 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -47,5 +47,4 @@ impl<K: Key, V: Value> AdmissionPolicy for AdmitAll<K, V> {
     }
 }
 
-pub mod all;
 pub mod rated_random;
diff --git a/foyer-storage/src/reinsertion.rs b/foyer-storage/src/reinsertion.rs
index b1e59e10..679c935e 100644
--- a/foyer-storage/src/reinsertion.rs
+++ b/foyer-storage/src/reinsertion.rs
@@ -12,8 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::marker::PhantomData;
-
 use foyer_common::code::{Key, Value};
 
 use std::fmt::Debug;
@@ -24,28 +22,3 @@ pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
 
     fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
 }
-
-#[derive(Debug)]
-pub struct ReinsertNone<K: Key, V: Value>(PhantomData<(K, V)>);
-
-impl<K: Key, V: Value> Default for ReinsertNone<K, V> {
-    fn default() -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<K: Key, V: Value> Clone for ReinsertNone<K, V> {
-    fn clone(&self) -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<K: Key, V: Value> ReinsertionPolicy for ReinsertNone<K, V> {
-    type Key = K;
-
-    type Value = V;
-
-    fn judge(&self, _key: &Self::Key, _value: &Self::Value) -> bool {
-        false
-    }
-}
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index a5fffb0d..71989fbc 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -639,13 +639,9 @@ pub mod tests {
 
     use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
 
-    use crate::{
-        admission::AdmitAll,
-        device::{
-            fs::{FsDevice, FsDeviceConfig},
-            io_buffer::AlignedAllocator,
-        },
-        reinsertion::ReinsertNone,
+    use crate::device::{
+        fs::{FsDevice, FsDeviceConfig},
+        io_buffer::AlignedAllocator,
     };
 
     use super::*;
@@ -681,8 +677,8 @@ pub mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            admissions: vec![Arc::new(AdmitAll::default())],
-            reinsertions: vec![Arc::new(ReinsertNone::default())],
+            admissions: vec![],
+            reinsertions: vec![],
             buffer_pool_size: 8 * MB,
             flushers: 1,
             reclaimers: 1,
@@ -725,8 +721,8 @@ pub mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            admissions: vec![Arc::new(AdmitAll::default())],
-            reinsertions: vec![Arc::new(ReinsertNone::default())],
+            admissions: vec![],
+            reinsertions: vec![],
             buffer_pool_size: 8 * MB,
             flushers: 1,
             reclaimers: 0,

From e502b30958944e20b17ec05073c24fd6c4890f42 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 7 Jul 2023 20:20:25 +0800
Subject: [PATCH 042/261] fix: recovery run flushers and reclaimers first (#59)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 71989fbc..79599e56 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -167,8 +167,6 @@ where
             _marker: PhantomData,
         });
 
-        store.recover(config.recover_concurrency).await?;
-
         let mut handles = vec![];
         handles.append(
             &mut flusher
@@ -195,6 +193,8 @@ where
         );
         store.handles.lock().append(&mut handles);
 
+        store.recover(config.recover_concurrency).await?;
+
         Ok(store)
     }
 

From 296c51d1f30fa21b931ab888eb47085f1ddf3e82 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 10 Jul 2023 14:06:49 +0800
Subject: [PATCH 043/261] chore: remove old metrics, reclaim drop indices first
 (#60)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/reclaimer.rs |   8 +--
 foyer/src/lib.rs               |   2 -
 foyer/src/metrics.rs           | 109 ---------------------------------
 3 files changed, 4 insertions(+), 115 deletions(-)
 delete mode 100644 foyer/src/metrics.rs

diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 71318eec..d77cda22 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -188,7 +188,10 @@ where
 
         let region = self.region_manager.region(&task.region_id);
 
-        // keep region totally exclusive while reclamation
+        // step 1: drop indices
+        let _indices = self.indices.take_region(&task.region_id);
+
+        // after drop indices and acquire exclusive lock, no writers or readers are supposed to access the region
         let guard = region.exclusive(false, false, false).await;
 
         tracing::trace!(
@@ -199,9 +202,6 @@ where
             guard.physical_readers()
         );
 
-        // step 1: drop indices
-        let _indices = self.indices.take_region(&task.region_id);
-
         // step 2: do reinsertion
         // TODO(MrCroxx): do reinsertion
 
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 494f718a..0392500f 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -15,8 +15,6 @@
 #![feature(trait_alias)]
 #![feature(pattern)]
 
-mod metrics;
-
 pub trait Weight {
     fn weight(&self) -> usize;
 }
diff --git a/foyer/src/metrics.rs b/foyer/src/metrics.rs
deleted file mode 100644
index 09f92107..00000000
--- a/foyer/src/metrics.rs
+++ /dev/null
@@ -1,109 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use prometheus::{
-    register_counter_vec_with_registry, register_gauge_with_registry,
-    register_histogram_vec_with_registry, register_int_counter_with_registry, Counter, CounterVec,
-    Gauge, Histogram, HistogramVec, IntCounter, Registry,
-};
-
-pub struct Metrics {
-    pub latency_insert: Histogram,
-    pub latency_get: Histogram,
-    pub latency_remove: Histogram,
-
-    pub latency_store: Histogram,
-    pub latency_load: Histogram,
-    pub latency_delete: Histogram,
-
-    pub bytes_store: Counter,
-    pub bytes_load: Counter,
-    pub bytes_delete: Counter,
-
-    pub cache_data_size: Gauge,
-
-    pub miss: IntCounter,
-
-    _latency: HistogramVec,
-    _bytes: CounterVec,
-}
-
-impl Default for Metrics {
-    fn default() -> Self {
-        Self::new(Registry::new())
-    }
-}
-
-impl Metrics {
-    pub fn new(registry: Registry) -> Self {
-        let latency = register_histogram_vec_with_registry!(
-            "foyer_latency",
-            "foyer latency",
-            &["op"],
-            vec![
-                0.0001, 0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75,
-                1.0
-            ],
-            registry
-        )
-        .unwrap();
-
-        let bytes =
-            register_counter_vec_with_registry!("foyer_bytes", "foyer bytes", &["op"], registry)
-                .unwrap();
-
-        let latency_insert = latency.with_label_values(&["insert"]);
-        let latency_get = latency.with_label_values(&["get"]);
-        let latency_remove = latency.with_label_values(&["remove"]);
-
-        let latency_store = latency.with_label_values(&["store"]);
-        let latency_load = latency.with_label_values(&["load"]);
-        let latency_delete = latency.with_label_values(&["delete"]);
-
-        let bytes_store = bytes.with_label_values(&["store"]);
-        let bytes_load = bytes.with_label_values(&["load"]);
-        let bytes_delete = bytes.with_label_values(&["delete"]);
-
-        let miss =
-            register_int_counter_with_registry!("foyer_cache_miss", "foyer cache miss", registry)
-                .unwrap();
-        let cache_data_size = register_gauge_with_registry!(
-            "foyer_cache_data_size",
-            "foyer cache data size",
-            registry
-        )
-        .unwrap();
-
-        Self {
-            latency_insert,
-            latency_get,
-            latency_remove,
-
-            latency_store,
-            latency_load,
-            latency_delete,
-
-            bytes_store,
-            bytes_load,
-            bytes_delete,
-
-            cache_data_size,
-
-            miss,
-
-            _latency: latency,
-            _bytes: bytes,
-        }
-    }
-}

From 4086fb991ff893f5e3fb0ee6edd365c160d7fcd8 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 10 Jul 2023 16:58:33 +0800
Subject: [PATCH 044/261] chore: remove unused and rename mod (#61)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .../src/device/{io_buffer.rs => allocator.rs}          |  0
 foyer-storage/src/device/fs.rs                         |  2 +-
 foyer-storage/src/device/mod.rs                        |  4 ++--
 foyer-storage/src/lib.rs                               |  2 +-
 foyer-storage/src/store.rs                             |  2 +-
 foyer/src/lib.rs                                       | 10 ----------
 6 files changed, 5 insertions(+), 15 deletions(-)
 rename foyer-storage/src/device/{io_buffer.rs => allocator.rs} (100%)

diff --git a/foyer-storage/src/device/io_buffer.rs b/foyer-storage/src/device/allocator.rs
similarity index 100%
rename from foyer-storage/src/device/io_buffer.rs
rename to foyer-storage/src/device/allocator.rs
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 85808564..59b184c1 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -22,9 +22,9 @@ use std::{
 use crate::region::RegionId;
 
 use super::{
+    allocator::AlignedAllocator,
     asyncify,
     error::{Error, Result},
-    io_buffer::AlignedAllocator,
     Device, IoBuf, IoBufMut,
 };
 use async_trait::async_trait;
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 2197b912..bb45d4eb 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -12,9 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+pub mod allocator;
 pub mod error;
 pub mod fs;
-pub mod io_buffer;
 
 use async_trait::async_trait;
 use error::Result;
@@ -88,7 +88,7 @@ where
 
 #[cfg(test)]
 pub mod tests {
-    use super::{io_buffer::AlignedAllocator, *};
+    use super::{allocator::AlignedAllocator, *};
 
     #[derive(Debug, Clone)]
     pub struct NullDevice(AlignedAllocator);
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index e758c6fb..c77dee9b 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -19,7 +19,7 @@
 #![feature(let_chains)]
 #![allow(clippy::type_complexity)]
 
-use device::io_buffer::AlignedAllocator;
+use device::allocator::AlignedAllocator;
 
 pub mod admission;
 pub mod device;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 79599e56..6cedcea0 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -640,8 +640,8 @@ pub mod tests {
     use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
 
     use crate::device::{
+        allocator::AlignedAllocator,
         fs::{FsDevice, FsDeviceConfig},
-        io_buffer::AlignedAllocator,
     };
 
     use super::*;
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 0392500f..0ea25c6d 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -15,16 +15,6 @@
 #![feature(trait_alias)]
 #![feature(pattern)]
 
-pub trait Weight {
-    fn weight(&self) -> usize;
-}
-
-impl Weight for Vec<u8> {
-    fn weight(&self) -> usize {
-        self.len()
-    }
-}
-
 pub use foyer_common as common;
 pub use foyer_intrusive as intrusive;
 pub use foyer_storage as storage;

From b1512ad97eddbeff65a8667a7a4b85e8ce01cfdc Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 11 Jul 2023 11:55:17 +0800
Subject: [PATCH 045/261] fix: raise error in bench metrics instead of panic
 (#62)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 51fe9bb1..9e7cacc9 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -374,11 +374,11 @@ async fn write(
 
         let time = Instant::now();
         store.insert(idx, data).await.unwrap();
-        metrics
-            .insert_lats
-            .write()
-            .record(time.elapsed().as_micros() as u64)
-            .expect("record out of range");
+        let lat = time.elapsed().as_micros() as u64;
+        if let Err(e) = metrics.insert_lats.write().record(lat) {
+            tracing::error!("metrics error: {:?}, value: {}", e, lat);
+        }
+
         metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
         metrics
             .insert_bytes
@@ -425,17 +425,13 @@ async fn read(
 
         if res.is_some() {
             assert_eq!(vec![idx as u8; entry_size], res.unwrap());
-            metrics
-                .get_hit_lats
-                .write()
-                .record(lat)
-                .expect("record out of range");
+            if let Err(e) = metrics.get_hit_lats.write().record(lat) {
+                tracing::error!("metrics error: {:?}, value: {}", e, lat);
+            }
         } else {
-            metrics
-                .get_miss_lats
-                .write()
-                .record(lat)
-                .expect("record out of range");
+            if let Err(e) = metrics.get_miss_lats.write().record(lat) {
+                tracing::error!("metrics error: {:?}, value: {}", e, lat);
+            }
             metrics.get_miss_ios.fetch_add(1, Ordering::Relaxed);
         }
         metrics.get_ios.fetch_add(1, Ordering::Relaxed);

From 2fe1e8249cf6fe1cd7507bcd80f629fe47fcf454 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 12 Jul 2023 12:22:16 +0800
Subject: [PATCH 046/261] feat: introduce intrusive hashmap (#63)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/code.rs                   |  23 ++
 foyer-intrusive/src/collections/hashmap.rs | 403 +++++++++++++++++++++
 foyer-intrusive/src/collections/mod.rs     |   1 +
 foyer-intrusive/src/core/adapter.rs        |   4 +-
 foyer-storage/src/flusher.rs               |  17 +-
 foyer-storage/src/region_manager.rs        |   4 +-
 6 files changed, 432 insertions(+), 20 deletions(-)
 create mode 100644 foyer-intrusive/src/collections/hashmap.rs

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 5cd298b2..1a88e014 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -88,7 +88,30 @@ macro_rules! impl_key {
     };
 }
 
+macro_rules! impl_value {
+    ($( $type:ty, )*) => {
+        paste! {
+            $(
+                impl Value for $type {
+                    fn serialized_len(&self) -> usize {
+                        std::mem::size_of::<$type>()
+                    }
+
+                    fn write(&self, mut buf: &mut [u8]) {
+                        buf.[< put_ $type>](*self)
+                    }
+
+                    fn read(mut buf: &[u8]) -> Self {
+                        buf.[< get_ $type>]()
+                    }
+                }
+            )*
+        }
+    };
+}
+
 for_all_primitives! { impl_key }
+for_all_primitives! { impl_value }
 
 impl Value for Vec<u8> {
     fn serialized_len(&self) -> usize {
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
new file mode 100644
index 00000000..0a22d15d
--- /dev/null
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -0,0 +1,403 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{marker::PhantomData, ptr::NonNull};
+
+use foyer_common::code::{Key, Value};
+use std::hash::Hasher;
+use twox_hash::XxHash64;
+
+use crate::{
+    core::{
+        adapter::{KeyAdapter, Link},
+        pointer::PointerOps,
+    },
+    intrusive_adapter,
+};
+
+use super::dlist::{DList, DListIter, DListIterMut, DListLink};
+
+#[derive(Debug, Default)]
+pub struct HashMapLink {
+    dlist_link: DListLink,
+}
+
+intrusive_adapter! { HashMapLinkAdapter = NonNull<HashMapLink>: HashMapLink { dlist_link: DListLink } }
+
+impl HashMapLink {
+    pub fn raw(&self) -> NonNull<HashMapLink> {
+        unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
+    }
+}
+
+unsafe impl Send for HashMapLink {}
+unsafe impl Sync for HashMapLink {}
+
+impl Link for HashMapLink {
+    fn is_linked(&self) -> bool {
+        self.dlist_link.is_linked()
+    }
+}
+
+pub struct HashMap<K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = HashMapLink>,
+{
+    slots: Vec<DList<HashMapLinkAdapter>>,
+
+    len: usize,
+
+    adapter: A,
+
+    _marker: PhantomData<V>,
+}
+
+impl<K, V, A> Drop for HashMap<K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = HashMapLink>,
+{
+    fn drop(&mut self) {
+        unsafe {
+            for slot in self.slots.iter_mut() {
+                let mut iter = slot.iter_mut();
+                iter.front();
+                while iter.is_valid() {
+                    let link = iter.remove().unwrap();
+                    let item = self.adapter.link2item(link.as_ptr());
+                    let _ = self.adapter.pointer_ops().from_raw(item);
+                }
+            }
+        }
+    }
+}
+
+impl<K, V, A> HashMap<K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = HashMapLink>,
+{
+    pub fn new(bits: usize) -> Self {
+        let mut slots = Vec::with_capacity(1 << bits);
+        for _ in 0..(1 << bits) {
+            slots.push(DList::new());
+        }
+        Self {
+            slots,
+            len: 0,
+            adapter: A::new(),
+            _marker: PhantomData,
+        }
+    }
+
+    pub fn insert(
+        &mut self,
+        ptr: <A::PointerOps as PointerOps>::Pointer,
+    ) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+        unsafe {
+            let item_new = self.adapter.pointer_ops().into_raw(ptr);
+            let link_new = NonNull::new_unchecked(self.adapter.item2link(item_new) as *mut A::Link);
+
+            let key_new = &*self.adapter.item2key(item_new);
+            let hash = self.hash_key(key_new);
+            let slot = (self.slots.len() - 1) & hash as usize;
+
+            let res = self.remove_inner(key_new, slot);
+
+            self.slots[slot].push_front(link_new);
+
+            res
+        }
+    }
+
+    pub fn remove(&mut self, key: &K) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+        unsafe {
+            let hash = self.hash_key(key);
+            let slot = (self.slots.len() - 1) & hash as usize;
+
+            self.remove_inner(key, slot)
+        }
+    }
+
+    pub fn lookup(&self, key: &K) -> Option<&<A::PointerOps as PointerOps>::Item> {
+        unsafe {
+            let hash = self.hash_key(key);
+            let slot = (self.slots.len() - 1) & hash as usize;
+
+            match self.lookup_inner(key, slot) {
+                Some(iter) => {
+                    let link = iter.get().unwrap() as *const _;
+                    let item = &*self.adapter.link2item(link);
+                    Some(item)
+                }
+                None => None,
+            }
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn iter(&self) -> HashMapIter<'_, K, V, A> {
+        HashMapIter::new(self)
+    }
+
+    /// # Safety
+    ///
+    /// there must be at most one matches in the slot
+    unsafe fn lookup_inner_mut(
+        &mut self,
+        key: &K,
+        slot: usize,
+    ) -> Option<DListIterMut<'_, HashMapLinkAdapter>> {
+        let mut iter = self.slots[slot].iter_mut();
+        iter.front();
+        while iter.is_valid() {
+            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
+            let ikey = &*self.adapter.item2key(item);
+            if ikey == key {
+                return Some(iter);
+            }
+            iter.next();
+        }
+        None
+    }
+
+    /// # Safety
+    ///
+    /// there must be at most one matches in the slot
+    unsafe fn lookup_inner(
+        &self,
+        key: &K,
+        slot: usize,
+    ) -> Option<DListIter<'_, HashMapLinkAdapter>> {
+        let mut iter = self.slots[slot].iter();
+        iter.front();
+        while iter.is_valid() {
+            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
+            let ikey = &*self.adapter.item2key(item);
+            if ikey == key {
+                return Some(iter);
+            }
+            iter.next();
+        }
+        None
+    }
+
+    /// # Safety
+    ///
+    /// there must be at most one matches in the slot
+    unsafe fn remove_inner(
+        &mut self,
+        key: &K,
+        slot: usize,
+    ) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+        match self.lookup_inner_mut(key, slot) {
+            Some(mut iter) => {
+                let link = iter.remove().unwrap();
+                let item = self.adapter.link2item(link.as_ptr());
+                let ptr = self.adapter.pointer_ops().from_raw(item);
+                Some(ptr)
+            }
+            None => None,
+        }
+    }
+
+    fn hash_key(&self, key: &K) -> u64 {
+        let mut hasher = XxHash64::default();
+        key.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+pub struct HashMapIter<'a, K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = HashMapLink>,
+{
+    slot: usize,
+    iters: Vec<DListIter<'a, HashMapLinkAdapter>>,
+
+    adapter: A,
+    _marker: PhantomData<(K, V)>,
+}
+
+impl<'a, K, V, A> HashMapIter<'a, K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = HashMapLink>,
+{
+    pub fn new(hashmap: &'a HashMap<K, V, A>) -> Self {
+        let mut iters = Vec::with_capacity(hashmap.slots.len());
+        for slot in hashmap.slots.iter() {
+            let iter = slot.iter();
+            iters.push(iter);
+        }
+        Self {
+            slot: 0,
+            iters,
+
+            adapter: A::new(),
+            _marker: PhantomData,
+        }
+    }
+
+    pub fn is_valid(&self) -> bool {
+        self.iters[self.slot].is_valid()
+    }
+
+    pub fn get(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+        self.iters[self.slot]
+            .get()
+            .map(|link| unsafe { &*(self.adapter.link2item(link.raw().as_ptr()) as *const _) })
+    }
+
+    pub fn get_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+        self.iters[self.slot]
+            .get()
+            .map(|link| unsafe { &mut *(self.adapter.link2item(link.raw().as_ptr()) as *mut _) })
+    }
+
+    /// Move to next.
+    ///
+    /// If iter is on tail, move to null.
+    /// If iter is on null, move to head.
+    pub fn next(&mut self) {
+        if self.iters[self.slot].is_valid() {
+            self.iters[self.slot].next();
+            if !self.iters[self.slot].is_valid() && self.slot + 1 < self.iters.len() {
+                self.slot += 1;
+                self.iters[self.slot].front();
+                debug_assert!(self.is_valid());
+            }
+        } else {
+            self.front();
+        }
+    }
+
+    /// Move to prev.
+    ///
+    /// If iter is on head, move to null.
+    /// If iter is on null, move to tail.
+    pub fn prev(&mut self) {
+        if self.iters[self.slot].is_valid() {
+            self.iters[self.slot].prev();
+            if !self.iters[self.slot].is_valid() && self.slot > 0 {
+                self.slot -= 1;
+                self.iters[self.slot].back();
+                debug_assert!(self.is_valid());
+            }
+        } else {
+            self.back();
+        }
+    }
+
+    /// Move to front.
+    pub fn front(&mut self) {
+        self.slot = 0;
+        self.iters[self.slot].front();
+    }
+
+    /// Move to back.
+    pub fn back(&mut self) {
+        self.slot = self.iters.len() - 1;
+        self.iters[self.slot].back();
+    }
+}
+
+// TODO(MrCroxx): Need more tests.
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use itertools::Itertools;
+
+    use crate::{intrusive_adapter, key_adapter};
+
+    use super::*;
+
+    #[derive(Debug)]
+    struct HashMapItem {
+        link: HashMapLink,
+
+        key: u64,
+        value: u64,
+    }
+
+    intrusive_adapter! { HashMapItemAdapter = Arc<HashMapItem>: HashMapItem { link: HashMapLink } }
+    key_adapter! { HashMapItemAdapter = HashMapItem { key: u64 } }
+
+    #[test]
+    fn test_hashmap_simple() {
+        let mut map = HashMap::<u64, u64, HashMapItemAdapter>::new(6);
+        let items = (0..128)
+            .map(|i| HashMapItem {
+                link: HashMapLink::default(),
+                key: i,
+                value: i,
+            })
+            .map(Arc::new)
+            .collect_vec();
+
+        for item in items.iter() {
+            map.insert(item.clone());
+        }
+
+        for i in 0..128 {
+            let item = map.lookup(&i).unwrap();
+            assert_eq!(item.key, i);
+            assert_eq!(item.value, i);
+        }
+
+        for i in 0..64 {
+            assert!(map.remove(&i).is_some());
+        }
+
+        for i in 0..64 {
+            assert!(map.lookup(&i).is_none());
+        }
+        for i in 64..128 {
+            let item = map.lookup(&i).unwrap();
+            assert_eq!(item.key, i);
+            assert_eq!(item.value, i);
+        }
+
+        for item in items.iter() {
+            map.insert(item.clone());
+        }
+        for i in 0..128 {
+            let item = map.lookup(&i).unwrap();
+            assert_eq!(item.key, i);
+            assert_eq!(item.value, i);
+        }
+
+        drop(map);
+
+        for item in items {
+            assert_eq!(Arc::strong_count(&item), 1);
+        }
+    }
+}
diff --git a/foyer-intrusive/src/collections/mod.rs b/foyer-intrusive/src/collections/mod.rs
index 4fe9e427..7470bd29 100644
--- a/foyer-intrusive/src/collections/mod.rs
+++ b/foyer-intrusive/src/collections/mod.rs
@@ -13,3 +13,4 @@
 //  limitations under the License.
 
 pub mod dlist;
+pub mod hashmap;
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 6f5778b6..00534096 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -243,7 +243,7 @@ macro_rules! key_adapter {
 
             unsafe fn item2key(
                 &self,
-                item: *const <Self::PointerOps as PointerOps>::Item,
+                item: *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item,
             ) -> *const Self::Key {
                 (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
             }
@@ -312,7 +312,7 @@ macro_rules! priority_adapter {
 
             unsafe fn item2priority(
                 &self,
-                item: *const <Self::PointerOps as PointerOps>::Item,
+                item: *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item,
             ) -> *const Self::Priority {
                 (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
             }
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index e5145741..3c53bb4f 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -167,8 +167,7 @@ where
 
         {
             // wait all physical readers (from previous version) and writers done
-            let guard = region.exclusive(false, true, false).await;
-            drop(guard);
+            let _ = region.exclusive(false, true, false).await;
         }
 
         tracing::trace!("[flusher] write region {} back to device", task.region_id);
@@ -193,19 +192,7 @@ where
         let buffer = {
             // step 2: detach buffer
             let mut guard = region.exclusive(false, false, true).await;
-
-            let buffer = guard.detach_buffer();
-
-            tracing::trace!(
-                "[flusher] region {}, writers: {}, buffered readers: {}, physical readers: {}",
-                region.id(),
-                guard.writers(),
-                guard.buffered_readers(),
-                guard.physical_readers()
-            );
-
-            drop(guard);
-            buffer
+            guard.detach_buffer()
         };
 
         tracing::trace!("[flusher] step 3");
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 43803921..9fa50b35 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -16,9 +16,7 @@ use std::{marker::PhantomData, sync::Arc};
 
 use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{
-    core::{adapter::Link, pointer::PointerOps},
-    eviction::EvictionPolicy,
-    intrusive_adapter, key_adapter, priority_adapter,
+    core::adapter::Link, eviction::EvictionPolicy, intrusive_adapter, key_adapter, priority_adapter,
 };
 use parking_lot::RwLock;
 use tokio::sync::RwLock as AsyncRwLock;

From 0792d142412890a6ae9eb9521f0c56669b9d6168 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 12 Jul 2023 15:32:27 +0800
Subject: [PATCH 047/261] feat: introduce basic memory cache framework (#64)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml                                 |   1 +
 foyer-common/src/code.rs                   |  12 +
 foyer-intrusive/src/collections/hashmap.rs |  33 ++-
 foyer-intrusive/src/eviction/fifo.rs       |  19 +-
 foyer-intrusive/src/eviction/lfu.rs        |  19 +-
 foyer-intrusive/src/eviction/lru.rs        |  19 +-
 foyer-intrusive/src/eviction/mod.rs        |   8 +-
 foyer-memory/Cargo.toml                    |  48 ++++
 foyer-memory/src/lib.rs                    | 289 +++++++++++++++++++++
 9 files changed, 437 insertions(+), 11 deletions(-)
 create mode 100644 foyer-memory/Cargo.toml
 create mode 100644 foyer-memory/src/lib.rs

diff --git a/Cargo.toml b/Cargo.toml
index 59544b15..d6f6620a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,7 @@ members = [
     "foyer",
     "foyer-common",
     "foyer-intrusive",
+    "foyer-memory",
     "foyer-storage",
     "foyer-storage-bench",
 ]
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 1a88e014..34d41865 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -29,6 +29,10 @@ pub trait Key:
     + Clone
     + std::fmt::Debug
 {
+    fn weight(&self) -> usize {
+        std::mem::size_of::<Self>()
+    }
+
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
     }
@@ -44,6 +48,10 @@ pub trait Key:
 
 #[allow(unused)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
+    fn weight(&self) -> usize {
+        std::mem::size_of::<Self>()
+    }
+
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
     }
@@ -114,6 +122,10 @@ for_all_primitives! { impl_key }
 for_all_primitives! { impl_value }
 
 impl Value for Vec<u8> {
+    fn weight(&self) -> usize {
+        self.len()
+    }
+
     fn serialized_len(&self) -> usize {
         self.len()
     }
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 0a22d15d..f6d8f871 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -118,9 +118,14 @@ where
             let slot = (self.slots.len() - 1) & hash as usize;
 
             let res = self.remove_inner(key_new, slot);
+            if res.is_some() {
+                self.len -= 1;
+            }
 
             self.slots[slot].push_front(link_new);
 
+            self.len += 1;
+
             res
         }
     }
@@ -130,7 +135,11 @@ where
             let hash = self.hash_key(key);
             let slot = (self.slots.len() - 1) & hash as usize;
 
-            self.remove_inner(key, slot)
+            let res = self.remove_inner(key, slot);
+            if res.is_some() {
+                self.len -= 1;
+            }
+            res
         }
     }
 
@@ -162,6 +171,25 @@ where
         HashMapIter::new(self)
     }
 
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`HashMap`].
+    pub unsafe fn remove_in_place(
+        &mut self,
+        link: NonNull<HashMapLink>,
+    ) -> <A::PointerOps as PointerOps>::Pointer {
+        assert!(link.as_ref().is_linked());
+        let item = self.adapter.link2item(link.as_ptr());
+        let key = &*self.adapter.item2key(item);
+        let hash = self.hash_key(key);
+        let slot = (self.slots.len() - 1) & hash as usize;
+        self.slots[slot]
+            .iter_mut_from_raw(link.as_ref().dlist_link.raw())
+            .remove();
+        self.len -= 1;
+        self.adapter.pointer_ops().from_raw(item)
+    }
+
     /// # Safety
     ///
     /// there must be at most one matches in the slot
@@ -394,6 +422,9 @@ mod tests {
             assert_eq!(item.value, i);
         }
 
+        unsafe { map.remove_in_place(items[0].link.raw()) };
+        assert!(map.lookup(&0).is_none());
+
         drop(map);
 
         for item in items {
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 846581de..8abc336c 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -114,6 +114,8 @@ where
     config: FifoConfig,
     total_ratio: usize,
 
+    len: usize,
+
     adapter: A,
 }
 
@@ -150,6 +152,8 @@ where
             config,
             total_ratio,
 
+            len: 0,
+
             adapter: A::new(),
         }
     }
@@ -167,6 +171,8 @@ where
             self.segments[priority.into()].push_back(link);
 
             self.rebalance();
+
+            self.len += 1;
         }
     }
 
@@ -188,12 +194,18 @@ where
 
             self.rebalance();
 
+            self.len -= 1;
+
             self.adapter.pointer_ops().from_raw(item)
         }
     }
 
     fn access(&mut self, _ptr: &<A::PointerOps as PointerOps>::Pointer) {}
 
+    fn len(&self) -> usize {
+        self.len
+    }
+
     fn rebalance(&mut self) {
         unsafe {
             let total: usize = self.segments.iter().map(|queue| queue.len()).sum();
@@ -350,7 +362,6 @@ where
     }
 
     fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
-        tracing::debug!("[lfu] insert {:?}", ptr);
         self.insert(ptr)
     }
 
@@ -358,15 +369,17 @@ where
         &mut self,
         ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
     ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
-        tracing::debug!("[lfu] remove {:?}", ptr);
         self.remove(ptr)
     }
 
     fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
-        tracing::debug!("[lfu] access {:?}", ptr);
         self.access(ptr)
     }
 
+    fn len(&self) -> usize {
+        self.len()
+    }
+
     fn iter(&self) -> Self::E<'_> {
         self.iter()
     }
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index c612b280..4beed1a7 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -149,6 +149,8 @@ where
     /// the counts are halved every time the max_window_len is hit
     frequencies: CMSketchUsize,
 
+    len: usize,
+
     config: LfuConfig,
 
     adapter: A,
@@ -187,6 +189,8 @@ where
             // A dummy size, will be updated later.
             frequencies: CMSketchUsize::new_with_size(1, 1),
 
+            len: 0,
+
             config,
 
             adapter: A::new(),
@@ -220,6 +224,8 @@ where
 
             // If the number of counters are too small for the cache size, double them.
             self.maybe_grow_access_counters();
+
+            self.len += 1;
         }
     }
 
@@ -235,6 +241,8 @@ where
 
             self.remove_from_lru(link);
 
+            self.len -= 1;
+
             self.adapter.pointer_ops().from_raw(item)
         }
     }
@@ -252,6 +260,10 @@ where
         }
     }
 
+    fn len(&self) -> usize {
+        self.len
+    }
+
     fn iter(&self) -> LfuIter<A> {
         let mut iter_main = self.lru_main.iter();
         let mut iter_tiny = self.lru_tiny.iter();
@@ -527,7 +539,6 @@ where
     }
 
     fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
-        tracing::debug!("[lfu] insert {:?}", ptr);
         self.insert(ptr)
     }
 
@@ -535,15 +546,17 @@ where
         &mut self,
         ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
     ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
-        tracing::debug!("[lfu] remove {:?}", ptr);
         self.remove(ptr)
     }
 
     fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
-        tracing::debug!("[lfu] access {:?}", ptr);
         self.access(ptr)
     }
 
+    fn len(&self) -> usize {
+        self.len()
+    }
+
     fn iter(&self) -> Self::E<'_> {
         self.iter()
     }
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 921b67cd..2a00466e 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -80,6 +80,8 @@ where
     /// length of tail after insertion point
     tail_len: usize,
 
+    len: usize,
+
     config: LruConfig,
 
     adapter: A,
@@ -114,6 +116,8 @@ where
 
             tail_len: 0,
 
+            len: 0,
+
             config,
 
             adapter: A::new(),
@@ -130,6 +134,8 @@ where
             self.insert_lru(link);
 
             self.update_lru_insertion_point();
+
+            self.len += 1;
         }
     }
 
@@ -153,6 +159,8 @@ where
                 self.tail_len -= 1;
             }
 
+            self.len -= 1;
+
             self.adapter.pointer_ops().from_raw(item)
         }
     }
@@ -176,6 +184,10 @@ where
         }
     }
 
+    fn len(&self) -> usize {
+        self.len
+    }
+
     fn iter(&self) -> LruIter<'_, A> {
         let mut iter = self.lru.iter();
         iter.back();
@@ -379,7 +391,6 @@ where
     }
 
     fn insert(&mut self, ptr: <<A>::PointerOps as PointerOps>::Pointer) {
-        tracing::debug!("[lru] insert {:?}", ptr);
         self.insert(ptr)
     }
 
@@ -387,15 +398,17 @@ where
         &mut self,
         ptr: &<<A>::PointerOps as PointerOps>::Pointer,
     ) -> <<A>::PointerOps as PointerOps>::Pointer {
-        tracing::debug!("[lru] remove {:?}", ptr);
         self.remove(ptr)
     }
 
     fn access(&mut self, ptr: &<<A>::PointerOps as PointerOps>::Pointer) {
-        tracing::debug!("[lru] access {:?}", ptr);
         self.access(ptr)
     }
 
+    fn len(&self) -> usize {
+        self.len()
+    }
+
     fn iter(&self) -> Self::E<'_> {
         self.iter()
     }
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index f3fce481..7889d625 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -19,7 +19,7 @@ use crate::core::{
 
 use std::fmt::Debug;
 
-pub trait Config = Send + Sync + 'static + Debug;
+pub trait Config = Send + Sync + 'static + Debug + Clone;
 
 pub trait EvictionPolicy<A>: Send + Sync + 'static
 where
@@ -41,6 +41,12 @@ where
 
     fn access(&mut self, ptr: &<A::PointerOps as PointerOps>::Pointer);
 
+    fn len(&self) -> usize;
+
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
     fn iter(&self) -> Self::E<'_>;
 
     fn push(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
new file mode 100644
index 00000000..5a83d42f
--- /dev/null
+++ b/foyer-memory/Cargo.toml
@@ -0,0 +1,48 @@
+[package]
+name = "foyer-memory"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-channel = "1.8"
+async-trait = "0.1"
+bitflags = "2.3.1"
+bytes = "1"
+cmsketch = "0.1"
+foyer-common = { path = "../foyer-common" }
+foyer-intrusive = { path = "../foyer-intrusive" }
+futures = "0.3"
+itertools = "0.10.5"
+libc = "0.2"
+memoffset = "0.8"
+nix = { version = "0.26", features = ["fs", "mman"] }
+parking_lot = "0.12"
+paste = "1.0"
+pin-project = "1"
+prometheus = "0.13"
+rand = "0.8.5"
+thiserror = "1"
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+tracing = "0.1"
+twox-hash = "1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+hdrhistogram = "7"
+rand_mt = "4.2.1"
+tempfile = "3"
+
+[features]
+deadlock = ["parking_lot/deadlock_detection"]
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
new file mode 100644
index 00000000..9b3aab6a
--- /dev/null
+++ b/foyer-memory/src/lib.rs
@@ -0,0 +1,289 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::Arc;
+
+use foyer_common::code::{Key, Value};
+use foyer_intrusive::{
+    collections::hashmap::{HashMap, HashMapLink},
+    core::adapter::Link,
+    eviction::EvictionPolicy,
+    intrusive_adapter, key_adapter, priority_adapter,
+};
+use parking_lot::Mutex;
+use std::hash::Hasher;
+use twox_hash::XxHash64;
+
+pub struct CacheConfig<K, V, E, EL>
+where
+    K: Key,
+    V: Value,
+    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
+    EL: Link,
+{
+    capacity: usize,
+    shard_bits: usize,
+    hashmap_bits: usize,
+    eviction_config: E::Config,
+}
+
+pub struct Cache<K, V, E, EL>
+where
+    K: Key,
+    V: Value,
+    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
+    EL: Link,
+{
+    shards: Vec<Mutex<CacheShard<K, V, E, EL>>>,
+}
+
+struct CacheShard<K, V, E, EL>
+where
+    K: Key,
+    V: Value,
+    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
+    EL: Link,
+{
+    container: HashMap<K, V, CacheItemHmAdapter<K, V, EL>>,
+    eviction: E,
+
+    capacity: usize,
+    size: usize,
+}
+
+#[derive(Debug)]
+pub struct CacheItem<K, V, EL>
+where
+    K: Key,
+    V: Value,
+    EL: Link,
+{
+    elink: EL,
+    clink: HashMapLink,
+
+    key: K,
+    value: V,
+
+    priority: usize,
+}
+
+impl<K, V, EL> CacheItem<K, V, EL>
+where
+    K: Key,
+    V: Value,
+    EL: Link,
+{
+    pub fn new(key: K, value: V) -> Self {
+        Self {
+            elink: EL::default(),
+            clink: HashMapLink::default(),
+            key,
+            value,
+
+            priority: 0,
+        }
+    }
+
+    pub fn key(&self) -> &K {
+        &self.key
+    }
+
+    pub fn value(&self) -> &V {
+        &self.value
+    }
+}
+
+intrusive_adapter! { pub CacheItemEpAdapter<K, V, EL> = Arc<CacheItem<K, V, EL>>: CacheItem<K, V, EL> { elink: EL } where K: Key, V: Value, EL: Link }
+key_adapter! { CacheItemEpAdapter<K, V, EL> = CacheItem<K, V, EL> { key: K } where K: Key, V: Value, EL: Link }
+priority_adapter! { CacheItemEpAdapter<K, V, EL> = CacheItem<K, V, EL> { priority: usize } where K: Key, V: Value, EL: Link }
+
+intrusive_adapter! { pub CacheItemHmAdapter<K, V, EL> = Arc<CacheItem<K, V, EL>>: CacheItem<K, V, EL> { clink: HashMapLink } where K: Key, V: Value, EL: Link }
+key_adapter! { CacheItemHmAdapter<K, V, EL> = CacheItem<K, V, EL> { key: K } where K: Key, V: Value, EL: Link }
+
+impl<K, V, E, EL> Cache<K, V, E, EL>
+where
+    K: Key,
+    V: Value,
+    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
+    EL: Link,
+{
+    pub fn new(config: CacheConfig<K, V, E, EL>) -> Self {
+        let mut shards = Vec::with_capacity(1 << config.shard_bits);
+
+        let shard_capacity = config.capacity / (1 << config.shard_bits);
+
+        for _ in 0..(1 << config.shard_bits) {
+            let shard = CacheShard {
+                container: HashMap::new(config.hashmap_bits),
+                eviction: E::new(config.eviction_config.clone()),
+                capacity: shard_capacity,
+                size: 0,
+            };
+            shards.push(Mutex::new(shard));
+        }
+
+        Self { shards }
+    }
+
+    pub fn insert(&self, key: K, value: V) -> Vec<Arc<CacheItem<K, V, EL>>> {
+        let hash = self.hash_key(&key);
+        let slot = (self.shards.len() - 1) & hash as usize;
+        let weight = key.weight() + value.weight();
+
+        let item = Arc::new(CacheItem::new(key, value));
+
+        let mut shard = self.shards[slot].lock();
+
+        let to_evict = {
+            let mut to_evict = vec![];
+            let mut to_evict_size = 0;
+            for item in shard.eviction.iter() {
+                if shard.size + weight - to_evict_size <= shard.capacity {
+                    break;
+                }
+                to_evict.push(item.clone());
+                to_evict_size += item.key.weight() + item.value.weight();
+            }
+            to_evict
+        };
+        for item in to_evict.iter() {
+            shard.eviction.remove(item);
+            unsafe { shard.container.remove_in_place(item.clink.raw()) };
+        }
+
+        shard.container.insert(item.clone());
+        shard.eviction.insert(item);
+        shard.size += weight;
+
+        to_evict
+    }
+
+    pub fn remove(&self, key: &K) -> Option<Arc<CacheItem<K, V, EL>>> {
+        let hash = self.hash_key(key);
+        let slot = (self.shards.len() - 1) & hash as usize;
+
+        let mut shard = self.shards[slot].lock();
+
+        let item = shard.container.remove(key);
+        if let Some(item) = &item {
+            shard.eviction.remove(item);
+        }
+        item
+    }
+
+    pub fn lookup(&self, key: &K) -> Option<Arc<CacheItem<K, V, EL>>> {
+        let hash = self.hash_key(key);
+        let slot = (self.shards.len() - 1) & hash as usize;
+
+        let mut shard = self.shards[slot].lock();
+        match shard.container.lookup(key) {
+            Some(item) => {
+                let item = unsafe {
+                    Arc::increment_strong_count(item as *const _);
+                    Arc::from_raw(item as *const _)
+                };
+                shard.eviction.access(&item);
+                Some(item)
+            }
+            None => None,
+        }
+    }
+
+    fn hash_key(&self, key: &K) -> u64 {
+        let mut hasher = XxHash64::default();
+        key.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
+
+    use super::*;
+
+    #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+    struct K(usize);
+
+    impl Key for K {
+        fn weight(&self) -> usize {
+            0
+        }
+    }
+
+    #[derive(Debug, PartialEq, Eq)]
+    struct V(usize);
+
+    impl Value for V {
+        fn weight(&self) -> usize {
+            self.0
+        }
+    }
+
+    type FifoCacheConfig = CacheConfig<K, V, Fifo<CacheItemEpAdapter<K, V, FifoLink>>, FifoLink>;
+    type FifoCache = Cache<K, V, Fifo<CacheItemEpAdapter<K, V, FifoLink>>, FifoLink>;
+
+    #[test]
+    fn test_fifo_cache_simple() {
+        let config = FifoCacheConfig {
+            capacity: 10,
+            shard_bits: 0, // 1 shard
+            hashmap_bits: 6,
+            eviction_config: FifoConfig {
+                segment_ratios: vec![1],
+            },
+        };
+        let cache = FifoCache::new(config);
+
+        // cache: 1, 2, 3, 4
+        for i in 1..=4 {
+            let evicted = cache.insert(K(i), V(i));
+            assert!(evicted.is_empty());
+        }
+        for i in 1..=4 {
+            let item = cache.lookup(&K(i)).unwrap();
+            assert_eq!(item.key(), &K(i));
+            assert_eq!(item.value(), &V(i));
+        }
+
+        // cache: 4, 5
+        // evicted: 1, 2, 3
+        let evicted = cache.insert(K(5), V(5));
+        assert_eq!(evicted.len(), 3);
+        for (i, item) in evicted.into_iter().enumerate() {
+            assert_eq!(Arc::strong_count(&item), 1);
+            assert_eq!(item.key().0, i + 1);
+            assert_eq!(item.value().0, i + 1);
+        }
+
+        // lookup: 5
+        // cache: 4
+        // removed: 5
+        let res = cache.lookup(&K(5)).unwrap();
+        let item = cache.remove(&K(5)).unwrap();
+        assert_eq!(item.key(), &K(5));
+        assert_eq!(item.value(), &V(5));
+        assert_eq!(Arc::strong_count(&item), 2);
+        drop(res);
+        assert_eq!(Arc::strong_count(&item), 1);
+
+        // cache: 10
+        // evicted: 4
+        let evicted = cache.insert(K(10), V(10));
+        assert_eq!(evicted.len(), 1);
+        assert_eq!(evicted[0].key(), &K(4));
+        assert_eq!(evicted[0].value(), &V(4));
+        assert_eq!(Arc::strong_count(&evicted[0]), 1);
+    }
+}

From 0172d33cb52487739fa732b142ec84a4895a2141 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 13 Jul 2023 11:43:33 +0800
Subject: [PATCH 048/261] feat: sleep 10ms while trying to acquire exclusive
 lock (#65)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/admission/rated_random.rs |  5 ++++-
 foyer-storage/src/region.rs                 | 15 +++++++++------
 foyer-storage/src/region_manager.rs         |  1 -
 foyer-storage/src/store.rs                  |  4 ++++
 4 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
index 539d24cd..31b71648 100644
--- a/foyer-storage/src/admission/rated_random.rs
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -59,7 +59,10 @@ where
     V: Value,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("DynamicRandom").finish()
+        f.debug_struct("DynamicRandom")
+            .field("rate", &self.rate)
+            .field("probability", &self.probability.load(Ordering::Relaxed))
+            .finish()
     }
 }
 
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 7e86fac6..cfe549d3 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -555,13 +555,16 @@ impl<A: BufferAllocator> ErwLock<A> {
         can_physical_read: bool,
     ) -> OwnedRwLockWriteGuard<RegionInner<A>> {
         loop {
-            let guard = self.inner.clone().write_owned().await;
-            let is_ready = (can_write || guard.writers == 0)
-                && (can_buffered_read || guard.buffered_readers == 0)
-                && (can_physical_read || guard.physical_readers == 0);
-            if is_ready {
-                return guard;
+            {
+                let guard = self.inner.clone().write_owned().await;
+                let is_ready = (can_write || guard.writers == 0)
+                    && (can_buffered_read || guard.buffered_readers == 0)
+                    && (can_physical_read || guard.physical_readers == 0);
+                if is_ready {
+                    return guard;
+                }
             }
+            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
         }
     }
 }
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 9fa50b35..e52a892d 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -199,7 +199,6 @@ where
         }
     }
 
-    #[tracing::instrument(skip(self))]
     pub fn clean_regions(&self) -> &AsyncQueue<RegionId> {
         &self.clean_regions
     }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 6cedcea0..b404eb93 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -208,6 +208,7 @@ where
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
         let _timer = self.metrics.latency_insert.start_timer();
 
@@ -257,6 +258,7 @@ where
         Ok(true)
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
 
@@ -300,6 +302,7 @@ where
         res
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn remove(&self, key: &K) {
         let _timer = self.metrics.latency_remove.start_timer();
 
@@ -335,6 +338,7 @@ where
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn recover(&self, concurrency: usize) -> Result<()> {
         tracing::info!("start store recovery");
 

From bd5be058d3cd8cb61bc1de93299ef755b5089e31 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 13 Jul 2023 15:19:49 +0800
Subject: [PATCH 049/261] feat: introduce thread-safe rate limiter (#68)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      |  15 ++--
 .github/workflows/main.yml         |  15 ++--
 .github/workflows/pull-request.yml |  15 ++--
 foyer-common/Cargo.toml            |   4 ++
 foyer-common/src/code.rs           |   4 +-
 foyer-common/src/lib.rs            |   1 +
 foyer-common/src/rate.rs           | 112 +++++++++++++++++++++++++++++
 foyer-storage-bench/src/utils.rs   |   4 +-
 foyer-storage/src/admission/mod.rs |   2 +-
 foyer-storage/src/device/fs.rs     |   2 +-
 rustfmt.toml                       |   3 +-
 11 files changed, 149 insertions(+), 28 deletions(-)
 create mode 100644 foyer-common/src/rate.rs

diff --git a/.github/template/template.yml b/.github/template/template.yml
index c96b3a3d..6ced2d33 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -59,8 +59,8 @@ jobs:
           cargo fmt --all -- --check
       - name: Run rust clippy check
         run: |
-          cargo clippy --all-targets --features tokio-console -- -D warnings &&
-          cargo clippy --all-targets --features deadlock -- -D warnings &&
+          cargo clippy --all-targets --features tokio-console -- -D warnings
+          cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -68,7 +68,8 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests::test_rated_random --exact --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
@@ -101,8 +102,8 @@ jobs:
           RUST_LOG: info
           TOKIO_WORKER_THREADS: 1
         run: |-
-          cargo build --all --features deadlock &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          cargo build --all --features deadlock
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
           timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
@@ -131,6 +132,6 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          cargo build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 926eeeee..5c1f8f9a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -66,8 +66,8 @@ jobs:
           cargo fmt --all -- --check
       - name: Run rust clippy check
         run: |
-          cargo clippy --all-targets --features tokio-console -- -D warnings &&
-          cargo clippy --all-targets --features deadlock -- -D warnings &&
+          cargo clippy --all-targets --features tokio-console -- -D warnings
+          cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -75,7 +75,8 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests::test_rated_random --exact --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
@@ -107,8 +108,8 @@ jobs:
           RUST_LOG: info
           TOKIO_WORKER_THREADS: 1
         run: |-
-          cargo build --all --features deadlock &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          cargo build --all --features deadlock
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
           timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
@@ -136,8 +137,8 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          cargo build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
 
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index eea70e28..fd5fd2c6 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -65,8 +65,8 @@ jobs:
           cargo fmt --all -- --check
       - name: Run rust clippy check
         run: |
-          cargo clippy --all-targets --features tokio-console -- -D warnings &&
-          cargo clippy --all-targets --features deadlock -- -D warnings &&
+          cargo clippy --all-targets --features tokio-console -- -D warnings
+          cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -74,7 +74,8 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests::test_rated_random --exact --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
@@ -106,8 +107,8 @@ jobs:
           RUST_LOG: info
           TOKIO_WORKER_THREADS: 1
         run: |-
-          cargo build --all --features deadlock &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          cargo build --all --features deadlock
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
           timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
@@ -135,8 +136,8 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          cargo build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
 concurrency:
   group: environment-${{ github.ref }}
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index eafaee68..45d70143 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -9,5 +9,9 @@ license = "Apache-2.0"
 
 [dependencies]
 bytes = "1"
+parking_lot = "0.12"
 paste = "1.0"
 tokio = { version = "1", features = ["sync"] }
+
+[dev-dependencies]
+rand = "0.8.5"
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 34d41865..60aaaf26 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -15,7 +15,7 @@
 use bytes::{Buf, BufMut};
 use paste::paste;
 
-#[allow(unused)]
+#[allow(unused_variables)]
 pub trait Key:
     Sized
     + Send
@@ -46,7 +46,7 @@ pub trait Key:
     }
 }
 
-#[allow(unused)]
+#[allow(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 2877c496..1a64d5ef 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -17,3 +17,4 @@
 pub mod bits;
 pub mod code;
 pub mod queue;
+pub mod rate;
diff --git a/foyer-common/src/rate.rs b/foyer-common/src/rate.rs
new file mode 100644
index 00000000..64f6397f
--- /dev/null
+++ b/foyer-common/src/rate.rs
@@ -0,0 +1,112 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::time::{Duration, Instant};
+
+use parking_lot::Mutex;
+
+pub struct RateLimiter {
+    inner: Mutex<Inner>,
+    rate: f64,
+}
+
+struct Inner {
+    quota: f64,
+
+    last: Instant,
+}
+
+impl RateLimiter {
+    pub fn new(rate: f64) -> Self {
+        let inner = Inner {
+            quota: 0.0,
+            last: Instant::now(),
+        };
+        Self {
+            rate,
+            inner: Mutex::new(inner),
+        }
+    }
+
+    pub fn consume(&self, weight: f64) -> Option<Duration> {
+        let mut inner = self.inner.lock();
+        let now = Instant::now();
+        let refill = now.duration_since(inner.last).as_secs_f64() * self.rate;
+        inner.last = now;
+        inner.quota = f64::min(inner.quota + refill, self.rate);
+        inner.quota -= weight;
+        if inner.quota >= 0.0 {
+            return None;
+        }
+        let wait = Duration::from_secs_f64((-inner.quota) / self.rate);
+        Some(wait)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    };
+
+    use rand::{thread_rng, Rng};
+
+    use super::*;
+
+    const ERATIO: f64 = 0.05;
+    const THREADS: usize = 8;
+    const RATE: usize = 1000;
+    const DURATION: Duration = Duration::from_secs(10);
+
+    #[ignore]
+    #[test]
+    fn test_rate_limiter() {
+        let v = Arc::new(AtomicUsize::new(0));
+        let limiter = Arc::new(RateLimiter::new(RATE as f64));
+        let task = |rate: usize, v: Arc<AtomicUsize>, limiter: Arc<RateLimiter>| {
+            let start = Instant::now();
+            loop {
+                if start.elapsed() >= DURATION {
+                    break;
+                }
+                if let Some(dur) = limiter.consume(rate as f64) {
+                    std::thread::sleep(dur);
+                }
+                v.fetch_add(rate, Ordering::Relaxed);
+            }
+        };
+        let mut handles = vec![];
+        let mut rng = thread_rng();
+        for _ in 0..THREADS {
+            let rate = rng.gen_range(10..20);
+            let handle = std::thread::spawn({
+                let v = v.clone();
+                let limiter = limiter.clone();
+                move || task(rate, v, limiter)
+            });
+            handles.push(handle);
+        }
+
+        for handle in handles {
+            handle.join().unwrap();
+        }
+
+        let error = (v.load(Ordering::Relaxed) as isize
+            - RATE as isize * DURATION.as_secs() as isize)
+            .unsigned_abs();
+        let eratio = error as f64 / (RATE as f64 * DURATION.as_secs_f64());
+        assert!(eratio < ERATIO, "eratio: {}, target: {}", eratio, ERATIO);
+    }
+}
diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
index d815da36..1344c5b1 100644
--- a/foyer-storage-bench/src/utils.rs
+++ b/foyer-storage-bench/src/utils.rs
@@ -31,7 +31,7 @@ use std::path::{Path, PathBuf};
 use itertools::Itertools;
 use nix::{fcntl::readlink, sys::stat::stat};
 
-#[allow(unused)]
+#[allow(unused_variables)]
 #[derive(PartialEq, Clone, Copy, Debug)]
 pub enum FsType {
     Xfs,
@@ -41,7 +41,7 @@ pub enum FsType {
     Others,
 }
 
-#[allow(unused)]
+#[allow(unused_variables)]
 pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
     #[cfg(target_os = "linux")]
     {
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 9b7ad6fe..dca37f26 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -18,7 +18,7 @@ use foyer_common::code::{Key, Value};
 
 use std::fmt::Debug;
 
-#[allow(unused)]
+#[allow(unused_variables)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 59b184c1..15f29ede 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -61,7 +61,7 @@ impl FsDeviceConfig {
 struct FsDeviceInner {
     config: FsDeviceConfig,
 
-    #[allow(unused)]
+    #[allow(unused_variables)]
     dir: File,
 
     files: Vec<File>,
diff --git a/rustfmt.toml b/rustfmt.toml
index d9ba5fdb..946ebb58 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1 +1,2 @@
-imports_granularity = "Crate"
\ No newline at end of file
+imports_granularity = "Crate"
+tab_spaces = 4
\ No newline at end of file

From 5ffe36df0602fbd477749395ea02354697496975 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 13 Jul 2023 15:47:10 +0800
Subject: [PATCH 050/261] feat: introduce flush & reclaim rate limit (#70)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs | 12 +++++++++++-
 foyer-storage/src/flusher.rs    |  9 ++++++++-
 foyer-storage/src/reclaimer.rs  |  5 +++++
 foyer-storage/src/store.rs      | 19 ++++++++++++++++++-
 4 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 9e7cacc9..d92d9938 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -117,9 +117,17 @@ pub struct Args {
     recover_concurrency: usize,
 
     /// enable rated random admission policy if `rated_random` > 0
-    /// (MiB)
+    /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     rated_random: usize,
+
+    /// (MiB/s)
+    #[arg(long, default_value_t = 0)]
+    flush_rate_limit: usize,
+
+    /// (MiB/s)
+    #[arg(long, default_value_t = 0)]
+    reclaim_rate_limit: usize,
 }
 
 impl Args {
@@ -225,7 +233,9 @@ async fn main() {
         reinsertions: vec![],
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
         flushers: args.flushers,
+        flush_rate_limit: args.flush_rate_limit * 1024 * 1024,
         reclaimers: args.reclaimers,
+        reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
         prometheus_registry: None,
     };
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 3c53bb4f..1efeb4d1 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use foyer_common::queue::AsyncQueue;
+use foyer_common::{queue::AsyncQueue, rate::RateLimiter};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use itertools::Itertools;
 use tokio::{
@@ -64,6 +64,7 @@ impl Flusher {
         &self,
         buffers: Arc<AsyncQueue<Vec<u8, A>>>,
         region_manager: Arc<RegionManager<A, D, E, EL>>,
+        rate_limiter: Option<Arc<RateLimiter>>,
         stop_rxs: Vec<broadcast::Receiver<()>>,
         metrics: Arc<Metrics>,
     ) -> Vec<JoinHandle<()>>
@@ -89,6 +90,7 @@ impl Flusher {
                 task_rx,
                 buffers: buffers.clone(),
                 region_manager: region_manager.clone(),
+                rate_limiter: rate_limiter.clone(),
                 stop_rx,
                 metrics: metrics.clone(),
             })
@@ -128,6 +130,8 @@ where
 
     region_manager: Arc<RegionManager<A, D, E, EL>>,
 
+    rate_limiter: Option<Arc<RateLimiter>>,
+
     stop_rx: broadcast::Receiver<()>,
 
     metrics: Arc<Metrics>,
@@ -179,6 +183,9 @@ where
             let end = std::cmp::min(offset + len, region.device().region_size());
 
             let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
+            if let Some(limiter) = &self.rate_limiter && let Some(duration) = limiter.consume(len as f64) {
+                tokio::time::sleep(duration).await;
+            }
             region
                 .device()
                 .write(s, region.id(), offset as u64, len)
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index d77cda22..2a05e199 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -28,6 +28,7 @@ use bytes::BufMut;
 use foyer_common::{
     code::{Key, Value},
     queue::AsyncQueue,
+    rate::RateLimiter,
 };
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use itertools::Itertools;
@@ -74,6 +75,7 @@ impl Reclaimer {
         clean_regions: Arc<AsyncQueue<RegionId>>,
         reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
         indices: Arc<Indices<K>>,
+        rate_limiter: Option<Arc<RateLimiter>>,
         stop_rxs: Vec<broadcast::Receiver<()>>,
         metrics: Arc<Metrics>,
     ) -> Vec<JoinHandle<()>>
@@ -104,6 +106,7 @@ impl Reclaimer {
                 clean_regions: clean_regions.clone(),
                 _reinsertions: reinsertions.clone(),
                 indices: indices.clone(),
+                _rate_limiter: rate_limiter.clone(),
                 stop_rx,
                 metrics: metrics.clone(),
             })
@@ -151,6 +154,8 @@ where
     _reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
     indices: Arc<Indices<K>>,
 
+    _rate_limiter: Option<Arc<RateLimiter>>,
+
     stop_rx: broadcast::Receiver<()>,
 
     metrics: Arc<Metrics>,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index b404eb93..e893d37d 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -15,7 +15,7 @@
 use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant};
 
 use bytes::{Buf, BufMut};
-use foyer_common::{bits, queue::AsyncQueue};
+use foyer_common::{bits, queue::AsyncQueue, rate::RateLimiter};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use itertools::Itertools;
 use parking_lot::Mutex;
@@ -53,7 +53,9 @@ where
     pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
     pub buffer_pool_size: usize,
     pub flushers: usize,
+    pub flush_rate_limit: usize,
     pub reclaimers: usize,
+    pub reclaim_rate_limit: usize,
     pub recover_concurrency: usize,
     pub prometheus_registry: Option<prometheus::Registry>,
 }
@@ -167,12 +169,22 @@ where
             _marker: PhantomData,
         });
 
+        let flush_rate_limiter = match config.flush_rate_limit {
+            0 => None,
+            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
+        };
+        let reclaim_rate_limiter = match config.reclaim_rate_limit {
+            0 => None,
+            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
+        };
+
         let mut handles = vec![];
         handles.append(
             &mut flusher
                 .run(
                     buffers,
                     region_manager.clone(),
+                    flush_rate_limiter,
                     flusher_stop_rxs,
                     metrics.clone(),
                 )
@@ -186,6 +198,7 @@ where
                     clean_regions,
                     config.reinsertions,
                     indices,
+                    reclaim_rate_limiter,
                     reclaimer_stop_rxs,
                     metrics,
                 )
@@ -685,7 +698,9 @@ pub mod tests {
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
             flushers: 1,
+            flush_rate_limit: 0,
             reclaimers: 1,
+            reclaim_rate_limit: 0,
             recover_concurrency: 2,
             prometheus_registry: None,
         };
@@ -729,7 +744,9 @@ pub mod tests {
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
             flushers: 1,
+            flush_rate_limit: 0,
             reclaimers: 0,
+            reclaim_rate_limit: 0,
             recover_concurrency: 2,
             prometheus_registry: None,
         };

From 8381f025d7c430fdba23b6f8c0949e531c732109 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 13 Jul 2023 18:23:08 +0800
Subject: [PATCH 051/261] feat: introduce intrusive duplicated hashmap (#71)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/collections/dlist.rs      |  59 +++
 .../src/collections/duplicated_hashmap.rs     | 461 ++++++++++++++++++
 foyer-intrusive/src/collections/mod.rs        |   1 +
 3 files changed, 521 insertions(+)
 create mode 100644 foyer-intrusive/src/collections/duplicated_hashmap.rs

diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index 4b160d10..1d644efb 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -30,6 +30,14 @@ impl DListLink {
     pub fn raw(&self) -> NonNull<DListLink> {
         unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
     }
+
+    pub fn prev(&self) -> Option<NonNull<DListLink>> {
+        self.prev
+    }
+
+    pub fn next(&self) -> Option<NonNull<DListLink>> {
+        self.next
+    }
 }
 
 unsafe impl Send for DListLink {}
@@ -98,6 +106,24 @@ where
         }
     }
 
+    pub fn front_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+        unsafe {
+            self.head
+                .map(|link| self.adapter.link2item(link.as_ptr()))
+                .map(|link| link as *mut _)
+                .map(|link| &mut *link)
+        }
+    }
+
+    pub fn back_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+        unsafe {
+            self.tail
+                .map(|link| self.adapter.link2item(link.as_ptr()))
+                .map(|link| link as *mut _)
+                .map(|link| &mut *link)
+        }
+    }
+
     pub fn push_front(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
         self.iter_mut().insert_after(ptr);
     }
@@ -163,6 +189,23 @@ where
             dlist: self,
         }
     }
+
+    /// # Safety
+    ///
+    /// `self` must be empty. `src` will be set empty after operation.
+    pub unsafe fn replace_with(&mut self, src: &mut DList<A>) {
+        debug_assert!(self.head.is_none());
+        debug_assert!(self.tail.is_none());
+        debug_assert_eq!(self.len, 0);
+
+        self.head = src.head;
+        self.tail = src.tail;
+        self.len = src.len;
+
+        src.head = None;
+        src.tail = None;
+        src.len = 0;
+    }
 }
 
 pub struct DListIter<'a, A>
@@ -221,6 +264,14 @@ where
     pub fn back(&mut self) {
         self.link = self.dlist.tail;
     }
+
+    pub fn is_front(&self) -> bool {
+        self.link == self.dlist.head
+    }
+
+    pub fn is_back(&self) -> bool {
+        self.link == self.dlist.tail
+    }
 }
 
 pub struct DListIterMut<'a, A>
@@ -406,6 +457,14 @@ where
         link.as_mut().prev = prev;
         link.as_mut().next = next;
     }
+
+    pub fn is_front(&self) -> bool {
+        self.link == self.dlist.head
+    }
+
+    pub fn is_back(&self) -> bool {
+        self.link == self.dlist.tail
+    }
 }
 
 impl<'a, A> Iterator for DListIter<'a, A>
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
new file mode 100644
index 00000000..ffdbb5c3
--- /dev/null
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -0,0 +1,461 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, marker::PhantomData, ptr::NonNull};
+
+use foyer_common::code::{Key, Value};
+use std::hash::Hasher;
+use twox_hash::XxHash64;
+
+use crate::{
+    core::{
+        adapter::{Adapter, KeyAdapter, Link},
+        pointer::PointerOps,
+    },
+    intrusive_adapter,
+};
+
+use super::dlist::{DList, DListIter, DListIterMut, DListLink};
+
+pub struct DuplicatedHashMapLink {
+    slot_link: DListLink,
+    group_link: DListLink,
+    group: DList<DuplicatedHashMapLinkGroupAdapter>,
+}
+
+impl Default for DuplicatedHashMapLink {
+    fn default() -> Self {
+        Self {
+            slot_link: DListLink::default(),
+            group_link: DListLink::default(),
+            group: DList::new(),
+        }
+    }
+}
+
+impl Debug for DuplicatedHashMapLink {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DuplicatedHashMapLink")
+            .field("slot_link", &self.slot_link)
+            .field("group_link", &self.group_link)
+            .finish()
+    }
+}
+
+intrusive_adapter! { DuplicatedHashMapLinkSlotAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { slot_link: DListLink } }
+intrusive_adapter! { DuplicatedHashMapLinkGroupAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { group_link: DListLink } }
+
+impl DuplicatedHashMapLink {
+    pub fn raw(&self) -> NonNull<DuplicatedHashMapLink> {
+        unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
+    }
+}
+
+unsafe impl Send for DuplicatedHashMapLink {}
+unsafe impl Sync for DuplicatedHashMapLink {}
+
+impl Link for DuplicatedHashMapLink {
+    fn is_linked(&self) -> bool {
+        self.group_link.is_linked()
+    }
+}
+
+pub struct DuplicatedHashMap<K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = DuplicatedHashMapLink>,
+{
+    slots: Vec<DList<DuplicatedHashMapLinkSlotAdapter>>,
+
+    len: usize,
+
+    adapter: A,
+
+    _marker: PhantomData<V>,
+}
+
+impl<K, V, A> Drop for DuplicatedHashMap<K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = DuplicatedHashMapLink>,
+{
+    fn drop(&mut self) {
+        unsafe {
+            for slot in self.slots.iter_mut() {
+                let mut iter_slot = slot.iter_mut();
+                iter_slot.front();
+                while iter_slot.is_valid() {
+                    let mut link_slot = iter_slot.remove().unwrap();
+                    let mut iter_group = link_slot.as_mut().group.iter_mut();
+                    iter_group.front();
+                    while iter_group.is_valid() {
+                        let link_group = iter_group.remove().unwrap();
+                        let item = self.adapter.link2item(link_group.as_ptr());
+                        let _ = self.adapter.pointer_ops().from_raw(item);
+                    }
+                }
+            }
+        }
+    }
+}
+
+impl<K, V, A> DuplicatedHashMap<K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: KeyAdapter<Key = K, Link = DuplicatedHashMapLink>,
+{
+    pub fn new(bits: usize) -> Self {
+        let mut slots = Vec::with_capacity(1 << bits);
+        for _ in 0..(1 << bits) {
+            slots.push(DList::new());
+        }
+        Self {
+            slots,
+            len: 0,
+            adapter: A::new(),
+            _marker: PhantomData,
+        }
+    }
+
+    pub fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item_new = self.adapter.pointer_ops().into_raw(ptr);
+            let mut link_new =
+                NonNull::new_unchecked(self.adapter.item2link(item_new) as *mut A::Link);
+
+            assert!(link_new.as_ref().group.is_empty());
+
+            let key_new = &*self.adapter.item2key(item_new);
+            let hash = self.hash_key(key_new);
+            let slot = (self.slots.len() - 1) & hash as usize;
+
+            match self.lookup_inner_mut(key_new, slot) {
+                Some(mut iter) => {
+                    let link = iter.get_mut().unwrap();
+                    link.group.push_back(link_new);
+                }
+                None => {
+                    self.slots[slot].push_front(link_new);
+                    link_new.as_mut().group.push_back(link_new);
+                }
+            }
+
+            self.len += 1;
+        }
+    }
+
+    pub fn remove(&mut self, key: &K) -> Vec<<A::PointerOps as PointerOps>::Pointer> {
+        unsafe {
+            let hash = self.hash_key(key);
+            let slot = (self.slots.len() - 1) & hash as usize;
+
+            match self.lookup_inner_mut(key, slot) {
+                Some(mut iter) => {
+                    let mut link = iter.remove().unwrap();
+                    let mut res = Vec::with_capacity(link.as_ref().group.len());
+                    let mut iter = link.as_mut().group.iter_mut();
+                    iter.front();
+                    while iter.is_valid() {
+                        let link = iter.remove().unwrap();
+                        debug_assert!(!link.as_ref().is_linked());
+                        let item = self.adapter.link2item(link.as_ptr());
+                        let ptr = self.adapter.pointer_ops().from_raw(item);
+                        res.push(ptr);
+                    }
+                    debug_assert!(link.as_ref().group.is_empty());
+                    self.len -= res.len();
+                    res
+                }
+                None => vec![],
+            }
+        }
+    }
+
+    pub fn lookup(&self, key: &K) -> Vec<&<A::PointerOps as PointerOps>::Item> {
+        unsafe {
+            let hash = self.hash_key(key);
+            let slot = (self.slots.len() - 1) & hash as usize;
+
+            match self.lookup_inner(key, slot) {
+                Some(iter) => {
+                    let link = iter.get().unwrap();
+                    let mut res = Vec::with_capacity(link.group.len());
+                    let mut iter = link.group.iter();
+                    iter.front();
+                    while iter.is_valid() {
+                        let link = iter.get().unwrap();
+                        let item = &*self.adapter.link2item(link as *const _);
+                        res.push(item);
+                        iter.next();
+                    }
+                    res
+                }
+                None => vec![],
+            }
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`HashMap`].
+    pub unsafe fn remove_in_place(
+        &mut self,
+        mut link: NonNull<DuplicatedHashMapLink>,
+    ) -> <A::PointerOps as PointerOps>::Pointer {
+        assert!(link.as_ref().is_linked());
+        let item = self.adapter.link2item(link.as_ptr());
+        let key = &*self.adapter.item2key(item);
+        let hash = self.hash_key(key);
+        let slot = (self.slots.len() - 1) & hash as usize;
+
+        if link.as_ref().slot_link.is_linked() {
+            // the removed item is the group header
+
+            // remove from slot list
+            self.slots[slot]
+                .iter_mut_from_raw(link.as_ref().slot_link.raw())
+                .remove();
+
+            // remove from group list
+            link.as_mut()
+                .group
+                .iter_mut_from_raw(link.as_ref().group_link.raw())
+                .remove();
+
+            // transfer group list and relink slot list if necessary
+            if let Some(header_new) = link.as_mut().group.front_mut() {
+                self.slots[slot].push_front(header_new.raw());
+                header_new.group.replace_with(&mut link.as_mut().group);
+            }
+        } else {
+            // the removed item is NOT the group header
+
+            // find header link
+            let header_link = {
+                let mut header = &link.as_ref().group_link;
+                while header.prev().is_some() {
+                    header = &*header.prev().unwrap().as_ptr();
+                }
+                let adapter = DuplicatedHashMapLinkGroupAdapter::new();
+                &mut *(adapter.link2item(header as *const _) as *mut DuplicatedHashMapLink)
+            };
+            debug_assert!(header_link.slot_link.is_linked());
+
+            // remove from group link
+            header_link
+                .group
+                .iter_mut_from_raw(link.as_ref().group_link.raw())
+                .remove();
+        }
+
+        self.len -= 1;
+
+        debug_assert!(!link.as_ref().slot_link.is_linked());
+        debug_assert!(!link.as_ref().group_link.is_linked());
+        debug_assert!(link.as_ref().group.is_empty());
+
+        self.adapter.pointer_ops().from_raw(item)
+    }
+
+    /// # Safety
+    ///
+    /// there must be at most one matches in the slot
+    unsafe fn lookup_inner_mut(
+        &mut self,
+        key: &K,
+        slot: usize,
+    ) -> Option<DListIterMut<'_, DuplicatedHashMapLinkSlotAdapter>> {
+        let mut iter = self.slots[slot].iter_mut();
+        iter.front();
+        while iter.is_valid() {
+            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
+            let ikey = &*self.adapter.item2key(item);
+            if ikey == key {
+                return Some(iter);
+            }
+            iter.next();
+        }
+        None
+    }
+
+    /// # Safety
+    ///
+    /// there must be at most one matches in the slot
+    unsafe fn lookup_inner(
+        &self,
+        key: &K,
+        slot: usize,
+    ) -> Option<DListIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
+        let mut iter = self.slots[slot].iter();
+        iter.front();
+        while iter.is_valid() {
+            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
+            let ikey = &*self.adapter.item2key(item);
+            if ikey == key {
+                return Some(iter);
+            }
+            iter.next();
+        }
+        None
+    }
+
+    fn hash_key(&self, key: &K) -> u64 {
+        let mut hasher = XxHash64::default();
+        key.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+// TODO(MrCroxx): Need more tests.
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use itertools::Itertools;
+
+    use crate::{intrusive_adapter, key_adapter};
+
+    use super::*;
+
+    #[derive(Debug)]
+    struct DuplicatedHashMapItem {
+        link: DuplicatedHashMapLink,
+
+        key: u64,
+        value: u64,
+    }
+
+    intrusive_adapter! { HashMapItemAdapter = Arc<DuplicatedHashMapItem>: DuplicatedHashMapItem { link: DuplicatedHashMapLink } }
+    key_adapter! { HashMapItemAdapter = DuplicatedHashMapItem { key: u64 } }
+
+    #[test]
+    fn test_duplicated_hashmap_simple() {
+        // 2 slots
+        let mut map = DuplicatedHashMap::<u64, u64, HashMapItemAdapter>::new(1);
+
+        // 8 * 8 items
+        let items = (0..8)
+            .map(|key| {
+                (0..8)
+                    .map(|value| {
+                        Arc::new(DuplicatedHashMapItem {
+                            link: DuplicatedHashMapLink::default(),
+                            key,
+                            value,
+                        })
+                    })
+                    .collect_vec()
+            })
+            .collect_vec();
+
+        // 8 * 8
+        for item in items.iter().flatten() {
+            map.insert(item.clone());
+        }
+        for item in items.iter().flatten() {
+            assert_eq!(Arc::strong_count(item), 2);
+        }
+        for key in 0..8 {
+            let res = map.lookup(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            assert_eq!(vs, (0..8).collect_vec());
+        }
+
+        // { 0 2 4 6 } * 8
+        for key in (0..8).skip(1).step_by(2) {
+            let res = map.remove(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            assert_eq!(vs, (0..8).collect_vec());
+        }
+        for key in (0..8).step_by(2) {
+            let res = map.lookup(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            assert_eq!(vs, (0..8).collect_vec());
+        }
+
+        // 8 * 8
+        for item in items.iter().skip(1).step_by(2).flatten() {
+            map.insert(item.clone());
+        }
+        for key in 0..8 {
+            let res = map.lookup(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            assert_eq!(vs, (0..8).collect_vec());
+        }
+
+        // remove group member in place
+        unsafe { map.remove_in_place(items[4][4].link.raw()) };
+        for key in 0..8 {
+            let res = map.lookup(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            if key == 4 {
+                assert_eq!(vs, vec![0, 1, 2, 3, 5, 6, 7])
+            } else {
+                assert_eq!(vs, (0..8).collect_vec());
+            }
+        }
+        // 8 * 8
+        map.insert(items[4][4].clone());
+        for key in 0..8 {
+            let res = map.lookup(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            assert_eq!(vs, (0..8).collect_vec());
+        }
+
+        // remove group head in place
+        unsafe { map.remove_in_place(items[4][0].link.raw()) };
+        for key in 0..8 {
+            let res = map.lookup(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            if key == 4 {
+                assert_eq!(vs, vec![1, 2, 3, 4, 5, 6, 7])
+            } else {
+                assert_eq!(vs, (0..8).collect_vec());
+            }
+        }
+        // 8 * 8
+        map.insert(items[4][0].clone());
+        for key in 0..8 {
+            let res = map.lookup(&key);
+            let mut vs = res.iter().map(|item| item.value).collect_vec();
+            vs.sort();
+            assert_eq!(vs, (0..8).collect_vec());
+        }
+
+        drop(map);
+
+        for item in items.into_iter().flatten() {
+            assert_eq!(Arc::strong_count(&item), 1);
+        }
+    }
+}
diff --git a/foyer-intrusive/src/collections/mod.rs b/foyer-intrusive/src/collections/mod.rs
index 7470bd29..265cf55d 100644
--- a/foyer-intrusive/src/collections/mod.rs
+++ b/foyer-intrusive/src/collections/mod.rs
@@ -13,4 +13,5 @@
 //  limitations under the License.
 
 pub mod dlist;
+pub mod duplicated_hashmap;
 pub mod hashmap;

From ef2999f6c5a2157cb65bef6145b7d581201641b4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 14 Jul 2023 11:48:57 +0800
Subject: [PATCH 052/261] feat: add prometheus namespace config (#72)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |  4 +--
 foyer-storage/src/indices.rs    |  6 ++++
 foyer-storage/src/metrics.rs    | 49 +++++++++++++++++++--------------
 foyer-storage/src/store.rs      | 33 ++++++++++++++++++----
 4 files changed, 64 insertions(+), 28 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index d92d9938..e94c16fd 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -35,7 +35,7 @@ use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
     admission::{rated_random::RatedRandom, AdmissionPolicy},
     device::fs::FsDeviceConfig,
-    store::StoreConfig,
+    store::{PrometheusConfig, StoreConfig},
     LfuFsStore,
 };
 use futures::future::join_all;
@@ -237,7 +237,7 @@ async fn main() {
         reclaimers: args.reclaimers,
         reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
-        prometheus_registry: None,
+        prometheus_config: PrometheusConfig::default(),
     };
 
     println!("{:#?}", config);
diff --git a/foyer-storage/src/indices.rs b/foyer-storage/src/indices.rs
index 37f8223c..cab2a4df 100644
--- a/foyer-storage/src/indices.rs
+++ b/foyer-storage/src/indices.rs
@@ -96,6 +96,12 @@ where
         inner.regions[slot.region as usize].remove(&slot.sequence)
     }
 
+    pub fn clear(&self) {
+        let mut inner = self.inner.write();
+        inner.slots.clear();
+        inner.regions.iter_mut().for_each(|region| region.clear());
+    }
+
     pub fn take_region(&self, region: &RegionId) -> Vec<Index<K>> {
         let mut inner = self.inner.write();
         let mut indices = BTreeMap::new();
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 0bdea154..c5b700a0 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -14,7 +14,8 @@
 
 use prometheus::{
     register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
-    register_int_gauge_with_registry, Histogram, IntCounter, IntGauge, Registry,
+    register_int_gauge_with_registry, Histogram, HistogramOpts, IntCounter, IntGauge, Opts,
+    Registry,
 };
 
 #[derive(Debug)]
@@ -41,25 +42,31 @@ impl Default for Metrics {
 
 impl Metrics {
     pub fn new() -> Self {
-        Self::with_registry(Registry::default())
+        Self::with_registry_namespace(Registry::default(), "")
+    }
+
+    pub fn with_namespace(namespace: impl ToString) -> Self {
+        Self::with_registry_namespace(Registry::default(), namespace)
     }
 
     pub fn with_registry(registry: Registry) -> Self {
-        let latency = register_histogram_vec_with_registry!(
-            "foyer_storage_latency",
-            "foyer storage latency",
-            &["op", "extra"],
-            vec![0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
-            registry
-        )
-        .unwrap();
-        let bytes = register_int_counter_vec_with_registry!(
-            "foyer_storage_bytes",
-            "foyer storage bytes",
-            &["op", "extra"],
-            registry
-        )
-        .unwrap();
+        Self::with_registry_namespace(registry, "")
+    }
+
+    pub fn with_registry_namespace(registry: Registry, namespace: impl ToString) -> Self {
+        let latency = {
+            let opts = HistogramOpts::new("foyer_storage_latency", "foyer storage latency")
+                .namespace(namespace.to_string())
+                .buckets(vec![
+                    0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0,
+                ]);
+            register_histogram_vec_with_registry!(opts, &["op", "extra"], registry).unwrap()
+        };
+        let bytes = {
+            let opts = Opts::new("foyer_storage_bytes", "foyer storage bytes")
+                .namespace(namespace.to_string());
+            register_int_counter_vec_with_registry!(opts, &["op", "extra"], registry).unwrap()
+        };
 
         let latency_insert = latency.with_label_values(&["insert", ""]);
         let latency_lookup_hit = latency.with_label_values(&["lookup", "hit"]);
@@ -72,9 +79,11 @@ impl Metrics {
         let bytes_reclaim = bytes.with_label_values(&["reclaim", ""]);
         let bytes_reinsert = bytes.with_label_values(&["reinsert", ""]);
 
-        let size =
-            register_int_gauge_with_registry!("foyer_storage_size", "foyer storage size", registry)
-                .unwrap();
+        let size = {
+            let opts = Opts::new("foyer_storage_size", "foyer storage size")
+                .namespace(namespace.to_string());
+            register_int_gauge_with_registry!(opts, registry).unwrap()
+        };
 
         Self {
             latency_insert,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index e893d37d..f7aee655 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -39,6 +39,12 @@ use std::hash::Hasher;
 
 const REGION_MAGIC: u64 = 0x19970327;
 
+#[derive(Debug, Default)]
+pub struct PrometheusConfig {
+    pub registry: Option<prometheus::Registry>,
+    pub namespace: Option<String>,
+}
+
 pub struct StoreConfig<K, V, D, EP, EL>
 where
     K: Key,
@@ -57,7 +63,7 @@ where
     pub reclaimers: usize,
     pub reclaim_rate_limit: usize,
     pub recover_concurrency: usize,
-    pub prometheus_registry: Option<prometheus::Registry>,
+    pub prometheus_config: PrometheusConfig,
 }
 
 impl<K, V, D, EP, EL> Debug for StoreConfig<K, V, D, EP, EL>
@@ -153,10 +159,18 @@ where
             .map(|_| stop_tx.subscribe())
             .collect_vec();
 
-        let metrics = match config.prometheus_registry {
-            Some(registry) => Arc::new(crate::metrics::Metrics::with_registry(registry)),
-            None => Arc::new(crate::metrics::Metrics::new()),
+        let metrics = match (
+            config.prometheus_config.registry,
+            config.prometheus_config.namespace,
+        ) {
+            (Some(registry), Some(namespace)) => {
+                Metrics::with_registry_namespace(registry, namespace)
+            }
+            (Some(registry), None) => Metrics::with_registry(registry),
+            (None, Some(namespace)) => Metrics::with_namespace(namespace),
+            (None, None) => Metrics::new(),
         };
+        let metrics = Arc::new(metrics);
 
         let store = Arc::new(Self {
             indices: indices.clone(),
@@ -322,6 +336,13 @@ where
         self.indices.remove(key);
     }
 
+    #[tracing::instrument(skip(self))]
+    pub fn clear(&self) {
+        let _timer = self.metrics.latency_remove.start_timer();
+
+        self.indices.clear();
+    }
+
     fn serialized_len(&self, key: &K, value: &V) -> usize {
         let unaligned =
             key.serialized_len() + value.serialized_len() + EntryFooter::serialized_len();
@@ -702,7 +723,7 @@ pub mod tests {
             reclaimers: 1,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            prometheus_registry: None,
+            prometheus_config: PrometheusConfig::default(),
         };
 
         let store = TestStore::open(config).await.unwrap();
@@ -748,7 +769,7 @@ pub mod tests {
             reclaimers: 0,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            prometheus_registry: None,
+            prometheus_config: PrometheusConfig::default(),
         };
         let store = TestStore::open(config).await.unwrap();
 

From 754a03822a6a74bef24394684c32045bf66825c1 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 15 Jul 2023 16:56:12 +0800
Subject: [PATCH 053/261] feat: introduce event listener (#74)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |  1 +
 foyer-storage/src/error.rs      |  8 ++++++
 foyer-storage/src/event.rs      | 45 ++++++++++++++++++++++++++++++
 foyer-storage/src/indices.rs    | 46 +++++++++++++++++++++++++------
 foyer-storage/src/lib.rs        |  1 +
 foyer-storage/src/reclaimer.rs  | 12 +++++++-
 foyer-storage/src/store.rs      | 49 +++++++++++++++++++++++++++++----
 7 files changed, 147 insertions(+), 15 deletions(-)
 create mode 100644 foyer-storage/src/event.rs

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index e94c16fd..c1215af2 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -237,6 +237,7 @@ async fn main() {
         reclaimers: args.reclaimers,
         reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
+        event_listeners: vec![],
         prometheus_config: PrometheusConfig::default(),
     };
 
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index 7a0b0f47..1821bb4b 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -22,6 +22,8 @@ pub enum Error {
     ChecksumMismatch { checksum: u64, expected: u64 },
     #[error("channel full")]
     ChannelFull,
+    #[error("event listener error: {0}")]
+    EventListener(#[from] Box<dyn std::error::Error + Send + Sync + 'static>),
     #[error("other error: {0}")]
     Other(String),
 }
@@ -31,6 +33,12 @@ impl Error {
         Self::Device(e)
     }
 
+    pub fn event_listener(
+        e: impl Into<Box<dyn std::error::Error + Send + Sync + 'static>>,
+    ) -> Self {
+        Self::EventListener(e.into())
+    }
+
     pub fn other(e: impl Into<Box<dyn std::error::Error>>) -> Self {
         Self::Other(e.into().to_string())
     }
diff --git a/foyer-storage/src/event.rs b/foyer-storage/src/event.rs
new file mode 100644
index 00000000..0d532c06
--- /dev/null
+++ b/foyer-storage/src/event.rs
@@ -0,0 +1,45 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use async_trait::async_trait;
+use foyer_common::code::{Key, Value};
+
+use crate::error::Result;
+
+#[allow(unused_variables)]
+#[async_trait]
+pub trait EventListener: Send + Sync + 'static {
+    type K: Key;
+    type V: Value;
+
+    async fn on_recover(&self, key: &Self::K) -> Result<()> {
+        Ok(())
+    }
+
+    async fn on_insert(&self, key: &Self::K) -> Result<()> {
+        Ok(())
+    }
+
+    async fn on_remove(&self, key: &Self::K) -> Result<()> {
+        Ok(())
+    }
+
+    async fn on_evict(&self, key: &Self::K) -> Result<()> {
+        Ok(())
+    }
+
+    async fn on_clear(&self) -> Result<()> {
+        Ok(())
+    }
+}
diff --git a/foyer-storage/src/indices.rs b/foyer-storage/src/indices.rs
index cab2a4df..800da36d 100644
--- a/foyer-storage/src/indices.rs
+++ b/foyer-storage/src/indices.rs
@@ -16,7 +16,7 @@ use std::collections::BTreeMap;
 
 use foyer_common::code::Key;
 use itertools::Itertools;
-use parking_lot::RwLock;
+use parking_lot::{RwLock, RwLockWriteGuard};
 
 use crate::region::{RegionId, Version};
 
@@ -48,6 +48,7 @@ where
 {
     slots: BTreeMap<K, Slot>,
     regions: Vec<BTreeMap<u32, Index<K>>>,
+    sequences: Vec<usize>,
 }
 
 #[derive(Debug)]
@@ -66,6 +67,7 @@ where
         let inner = IndicesInner {
             slots: BTreeMap::new(),
             regions: vec![BTreeMap::new(); regions],
+            sequences: vec![0; regions],
         };
         Self {
             inner: RwLock::new(inner),
@@ -73,13 +75,8 @@ where
     }
 
     pub fn insert(&self, index: Index<K>) {
-        let region = index.region;
-        let key = index.key.clone();
-
         let mut inner = self.inner.write();
-        let sequence = inner.regions[region as usize].len() as u32;
-        inner.regions[region as usize].insert(sequence, index);
-        inner.slots.insert(key, Slot { region, sequence });
+        self.insert_inner(&mut inner, index)
     }
 
     pub fn lookup(&self, key: &K) -> Option<Index<K>> {
@@ -90,10 +87,21 @@ where
             .cloned()
     }
 
+    pub fn remap(&self, old_key: &K, new_key: K) -> bool {
+        let mut inner = self.inner.write();
+        match self.remove_inner(&mut inner, old_key) {
+            Some(mut index) => {
+                index.key = new_key;
+                self.insert_inner(&mut inner, index);
+                true
+            }
+            None => false,
+        }
+    }
+
     pub fn remove(&self, key: &K) -> Option<Index<K>> {
         let mut inner = self.inner.write();
-        let slot = inner.slots.remove(key)?;
-        inner.regions[slot.region as usize].remove(&slot.sequence)
+        self.remove_inner(&mut inner, key)
     }
 
     pub fn clear(&self) {
@@ -113,4 +121,24 @@ where
 
         indices.into_values().collect_vec()
     }
+
+    fn insert_inner(&self, inner: &mut RwLockWriteGuard<'_, IndicesInner<K>>, index: Index<K>) {
+        let region = index.region;
+        let key = index.key.clone();
+
+        let sequence = inner.sequences[region as usize] as u32;
+        inner.sequences[region as usize] += 1;
+
+        inner.regions[region as usize].insert(sequence, index);
+        inner.slots.insert(key, Slot { region, sequence });
+    }
+
+    fn remove_inner(
+        &self,
+        inner: &mut RwLockWriteGuard<'_, IndicesInner<K>>,
+        key: &K,
+    ) -> Option<Index<K>> {
+        let slot = inner.slots.remove(key)?;
+        inner.regions[slot.region as usize].remove(&slot.sequence)
+    }
 }
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index c77dee9b..883d794c 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -24,6 +24,7 @@ use device::allocator::AlignedAllocator;
 pub mod admission;
 pub mod device;
 pub mod error;
+pub mod event;
 pub mod flusher;
 pub mod indices;
 pub mod metrics;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 2a05e199..3440dcda 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -17,6 +17,7 @@ use std::sync::Arc;
 use crate::{
     device::{BufferAllocator, Device},
     error::{Error, Result},
+    event::EventListener,
     indices::Indices,
     metrics::Metrics,
     region::RegionId,
@@ -76,6 +77,7 @@ impl Reclaimer {
         reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
         indices: Arc<Indices<K>>,
         rate_limiter: Option<Arc<RateLimiter>>,
+        event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
         stop_rxs: Vec<broadcast::Receiver<()>>,
         metrics: Arc<Metrics>,
     ) -> Vec<JoinHandle<()>>
@@ -109,6 +111,7 @@ impl Reclaimer {
                 _rate_limiter: rate_limiter.clone(),
                 stop_rx,
                 metrics: metrics.clone(),
+                event_listeners: event_listeners.clone(),
             })
             .collect_vec();
 
@@ -156,6 +159,8 @@ where
 
     _rate_limiter: Option<Arc<RateLimiter>>,
 
+    event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
+
     stop_rx: broadcast::Receiver<()>,
 
     metrics: Arc<Metrics>,
@@ -194,7 +199,12 @@ where
         let region = self.region_manager.region(&task.region_id);
 
         // step 1: drop indices
-        let _indices = self.indices.take_region(&task.region_id);
+        let indices = self.indices.take_region(&task.region_id);
+        for index in indices.iter() {
+            for listener in self.event_listeners.iter() {
+                listener.on_evict(&index.key).await?;
+            }
+        }
 
         // after drop indices and acquire exclusive lock, no writers or readers are supposed to access the region
         let guard = region.exclusive(false, false, false).await;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index f7aee655..c26ecaba 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -26,6 +26,7 @@ use crate::{
     admission::AdmissionPolicy,
     device::{BufferAllocator, Device},
     error::{Error, Result},
+    event::EventListener,
     flusher::Flusher,
     indices::{Index, Indices},
     metrics::Metrics,
@@ -63,6 +64,7 @@ where
     pub reclaimers: usize,
     pub reclaim_rate_limit: usize,
     pub recover_concurrency: usize,
+    pub event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
     pub prometheus_config: PrometheusConfig,
 }
 
@@ -104,6 +106,8 @@ where
 
     admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
 
+    event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
+
     handles: Mutex<Vec<JoinHandle<()>>>,
 
     stop_tx: broadcast::Sender<()>,
@@ -177,6 +181,7 @@ where
             region_manager: region_manager.clone(),
             device: device.clone(),
             admissions: config.admissions,
+            event_listeners: config.event_listeners.clone(),
             handles: Mutex::new(vec![]),
             stop_tx,
             metrics: metrics.clone(),
@@ -213,6 +218,7 @@ where
                     config.reinsertions,
                     indices,
                     reclaim_rate_limiter,
+                    config.event_listeners,
                     reclaimer_stop_rxs,
                     metrics,
                 )
@@ -275,16 +281,25 @@ where
             key_len: key.serialized_len() as u32,
             value_len: value.serialized_len() as u32,
 
-            key,
+            key: key.clone(),
         };
 
         slice.destroy().await;
 
         self.indices.insert(index);
 
+        for listener in self.event_listeners.iter() {
+            listener.on_insert(&key).await?;
+        }
+
         Ok(true)
     }
 
+    #[tracing::instrument(skip(self))]
+    pub fn exists(&self, key: &K) -> Result<bool> {
+        Ok(self.indices.lookup(key).is_some())
+    }
+
     #[tracing::instrument(skip(self))]
     pub async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
@@ -330,17 +345,33 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn remove(&self, key: &K) {
+    pub async fn remove(&self, key: &K) -> Result<bool> {
         let _timer = self.metrics.latency_remove.start_timer();
 
-        self.indices.remove(key);
+        let res = self.indices.remove(key).is_some();
+
+        if res {
+            for listener in self.event_listeners.iter() {
+                listener.on_remove(key).await?;
+            }
+        }
+
+        Ok(res)
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn clear(&self) {
+    pub async fn clear(&self) -> Result<()> {
         let _timer = self.metrics.latency_remove.start_timer();
 
         self.indices.clear();
+
+        for listener in self.event_listeners.iter() {
+            listener.on_clear().await?;
+        }
+
+        // TODO(MrCroxx): set all regions as clean?
+
+        Ok(())
     }
 
     fn serialized_len(&self, key: &K, value: &V) -> usize {
@@ -384,9 +415,11 @@ where
             let irx = rx.clone();
             let region_manager = self.region_manager.clone();
             let indices = self.indices.clone();
+            let event_listeners = self.event_listeners.clone();
             let handle = tokio::spawn(async move {
                 itx.send(()).await.unwrap();
-                let res = Self::recover_region(region_id, region_manager, indices).await;
+                let res =
+                    Self::recover_region(region_id, region_manager, indices, event_listeners).await;
                 irx.recv().await.unwrap();
                 res
             });
@@ -411,10 +444,14 @@ where
         region_id: RegionId,
         region_manager: Arc<RegionManager<BA, D, EP, EL>>,
         indices: Arc<Indices<K>>,
+        event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
     ) -> Result<bool> {
         let region = region_manager.region(&region_id).clone();
         let res = if let Some(mut iter) = RegionEntryIter::<K, V, BA, D>::open(region).await? {
             while let Some(index) = iter.next().await? {
+                for listener in event_listeners.iter() {
+                    listener.on_recover(&index.key).await?;
+                }
                 indices.insert(index);
             }
             region_manager.set_region_evictable(&region_id).await;
@@ -723,6 +760,7 @@ pub mod tests {
             reclaimers: 1,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
+            event_listeners: vec![],
             prometheus_config: PrometheusConfig::default(),
         };
 
@@ -769,6 +807,7 @@ pub mod tests {
             reclaimers: 0,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
+            event_listeners: vec![],
             prometheus_config: PrometheusConfig::default(),
         };
         let store = TestStore::open(config).await.unwrap();

From 6c42c7f56434ca693dde76f163010bf4e6d6d7ce Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 17 Jul 2023 15:27:44 +0800
Subject: [PATCH 054/261] feat: introduce store writer (#75)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/admission/mod.rs          |  27 +--
 foyer-storage/src/admission/rated_random.rs |  25 +-
 foyer-storage/src/metrics.rs                |   9 +-
 foyer-storage/src/store.rs                  | 243 ++++++++++++++++----
 4 files changed, 219 insertions(+), 85 deletions(-)

diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index dca37f26..0be895b7 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -12,39 +12,20 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::marker::PhantomData;
-
+use async_trait::async_trait;
 use foyer_common::code::{Key, Value};
 
 use std::fmt::Debug;
 
 #[allow(unused_variables)]
+#[async_trait]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
-
-    fn admit(&self, key: &Self::Key, value: &Self::Value) {}
-}
-
-#[derive(Debug)]
-pub struct AdmitAll<K: Key, V: Value>(PhantomData<(K, V)>);
-
-impl<K: Key, V: Value> Default for AdmitAll<K, V> {
-    fn default() -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<K: Key, V: Value> AdmissionPolicy for AdmitAll<K, V> {
-    type Key = K;
-
-    type Value = V;
+    async fn judge(&self, key: &Self::Key, weight: usize) -> bool;
 
-    fn judge(&self, _key: &Self::Key, _value: &Self::Value) -> bool {
-        true
-    }
+    async fn admit(&self, key: &Self::Key, weight: usize);
 }
 
 pub mod rated_random;
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
index 31b71648..7bb714d5 100644
--- a/foyer-storage/src/admission/rated_random.rs
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -19,6 +19,7 @@ use std::{
     time::{Duration, Instant},
 };
 
+use async_trait::async_trait;
 use foyer_common::code::{Key, Value};
 use parking_lot::{Mutex, MutexGuard};
 use rand::{thread_rng, Rng};
@@ -96,18 +97,18 @@ where
         }
     }
 
-    fn judge(&self, key: &K, value: &V) -> bool {
+    fn judge(&self, _key: &K, weight: usize) -> bool {
         if let Some(inner) = self.inner.try_lock() {
             self.update(inner);
         }
 
-        // TODO(MrCroxx): unify weighter?
-        let weight = key.serialized_len() + value.serialized_len();
         self.bytes.fetch_add(weight, Ordering::Relaxed);
 
         thread_rng().gen_range(0..PRECISION) < self.probability.load(Ordering::Relaxed)
     }
 
+    fn admit(&self, _key: &K, _weight: usize) {}
+
     fn update(&self, mut inner: MutexGuard<'_, Inner<K, V>>) {
         let now = Instant::now();
 
@@ -132,6 +133,7 @@ where
     }
 }
 
+#[async_trait]
 impl<K, V> AdmissionPolicy for RatedRandom<K, V>
 where
     K: Key,
@@ -141,8 +143,12 @@ where
 
     type Value = V;
 
-    fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool {
-        self.judge(key, value)
+    async fn judge(&self, key: &Self::Key, weight: usize) -> bool {
+        self.judge(key, weight)
+    }
+
+    async fn admit(&self, key: &Self::Key, weight: usize) {
+        self.admit(key, weight)
     }
 }
 
@@ -168,9 +174,10 @@ mod tests {
         async fn submit(rr: Arc<RatedRandom<u64, Vec<u8>>>, score: Arc<AtomicUsize>) {
             loop {
                 tokio::time::sleep(Duration::from_millis(10)).await;
-                let size = thread_rng().gen_range(1000..10000);
-                if rr.judge(&0, &vec![0; size]) {
-                    score.fetch_add(size, Ordering::Relaxed);
+                let weight = thread_rng().gen_range(1000..10000);
+                if rr.judge(&0, weight) {
+                    score.fetch_add(weight, Ordering::Relaxed);
+                    rr.admit(&0, weight);
                 }
             }
         }
@@ -184,6 +191,6 @@ mod tests {
         let error = (s as isize - RATE as isize * 10).unsigned_abs();
         let eratio = error as f64 / (RATE as f64 * 10.0);
 
-        assert!(eratio < ERATIO);
+        assert!(eratio < ERATIO, "eratio: {} < ERATIO: {}", eratio, ERATIO);
     }
 }
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index c5b700a0..ad36fef2 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -20,7 +20,8 @@ use prometheus::{
 
 #[derive(Debug)]
 pub struct Metrics {
-    pub latency_insert: Histogram,
+    pub latency_insert_admitted: Histogram,
+    pub latency_insert_rejected: Histogram,
     pub latency_lookup_hit: Histogram,
     pub latency_lookup_miss: Histogram,
     pub latency_remove: Histogram,
@@ -68,7 +69,8 @@ impl Metrics {
             register_int_counter_vec_with_registry!(opts, &["op", "extra"], registry).unwrap()
         };
 
-        let latency_insert = latency.with_label_values(&["insert", ""]);
+        let latency_insert_admitted = latency.with_label_values(&["insert", "admitted"]);
+        let latency_insert_rejected = latency.with_label_values(&["insert", "rejected"]);
         let latency_lookup_hit = latency.with_label_values(&["lookup", "hit"]);
         let latency_lookup_miss = latency.with_label_values(&["lookup", "miss"]);
         let latency_remove = latency.with_label_values(&["remove", ""]);
@@ -86,7 +88,8 @@ impl Metrics {
         };
 
         Self {
-            latency_insert,
+            latency_insert_admitted,
+            latency_insert_rejected,
             latency_lookup_hit,
             latency_lookup_miss,
             latency_remove,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index c26ecaba..535ee6d4 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -12,7 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant};
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    sync::Arc,
+    time::{Duration, Instant},
+};
 
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, queue::AsyncQueue, rate::RateLimiter};
@@ -243,56 +248,15 @@ where
 
     #[tracing::instrument(skip(self))]
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
-        let _timer = self.metrics.latency_insert.start_timer();
-
-        for admission in &self.admissions {
-            if !admission.judge(&key, &value) {
-                return Ok(false);
-            }
-        }
-        for admission in &self.admissions {
-            admission.admit(&key, &value);
-        }
-
-        let serialized_len = self.serialized_len(&key, &value);
-        self.metrics.bytes_insert.inc_by(serialized_len as u64);
-
-        let mut slice = match self.region_manager.allocate(serialized_len).await {
-            crate::region::AllocateResult::Ok(slice) => slice,
-            crate::region::AllocateResult::Full { mut slice, remain } => {
-                // current region is full, write region footer and try allocate again
-                let footer = RegionFooter {
-                    magic: REGION_MAGIC,
-                    padding: remain as u64,
-                };
-                footer.write(slice.as_mut());
-                slice.destroy().await;
-                self.region_manager.allocate(serialized_len).await.unwrap()
-            }
-        };
-
-        write_entry(slice.as_mut(), &key, &value);
-
-        let index = Index {
-            region: slice.region_id(),
-            version: slice.version(),
-            offset: slice.offset() as u32,
-            len: slice.len() as u32,
-            key_len: key.serialized_len() as u32,
-            value_len: value.serialized_len() as u32,
-
-            key: key.clone(),
-        };
-
-        slice.destroy().await;
-
-        self.indices.insert(index);
-
-        for listener in self.event_listeners.iter() {
-            listener.on_insert(&key).await?;
-        }
+        let weight = self.serialized_len(&key, &value);
+        let writer = StoreWriter::new(self, key, weight);
+        writer.finish(value).await
+    }
 
-        Ok(true)
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self))]
+    pub fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, BA, D, EP, EL> {
+        StoreWriter::new(self, key, weight)
     }
 
     #[tracing::instrument(skip(self))]
@@ -462,6 +426,185 @@ where
         };
         Ok(res)
     }
+
+    async fn judge_inner(&self, writer: &StoreWriter<'_, K, V, BA, D, EP, EL>) -> bool {
+        for admission in &self.admissions {
+            if !admission.judge(&writer.key, writer.weight).await {
+                return false;
+            }
+        }
+        true
+    }
+
+    async fn apply_writer(
+        &self,
+        mut writer: StoreWriter<'_, K, V, BA, D, EP, EL>,
+        value: V,
+    ) -> Result<bool> {
+        let now = Instant::now();
+
+        if !writer.judge().await {
+            let duration = now.elapsed() + writer.duration;
+            self.metrics
+                .latency_insert_rejected
+                .observe(duration.as_secs_f64());
+            return Ok(false);
+        }
+
+        assert!(writer.admitted.unwrap());
+
+        let key = &writer.key;
+
+        for admission in &self.admissions {
+            admission.admit(key, writer.weight).await;
+        }
+
+        let serialized_len = self.serialized_len(key, &value);
+        assert_eq!(serialized_len, writer.weight);
+
+        self.metrics.bytes_insert.inc_by(serialized_len as u64);
+
+        let mut slice = match self.region_manager.allocate(serialized_len).await {
+            crate::region::AllocateResult::Ok(slice) => slice,
+            crate::region::AllocateResult::Full { mut slice, remain } => {
+                // current region is full, write region footer and try allocate again
+                let footer = RegionFooter {
+                    magic: REGION_MAGIC,
+                    padding: remain as u64,
+                };
+                footer.write(slice.as_mut());
+                slice.destroy().await;
+                self.region_manager.allocate(serialized_len).await.unwrap()
+            }
+        };
+
+        write_entry(slice.as_mut(), key, &value);
+
+        let index = Index {
+            region: slice.region_id(),
+            version: slice.version(),
+            offset: slice.offset() as u32,
+            len: slice.len() as u32,
+            key_len: key.serialized_len() as u32,
+            value_len: value.serialized_len() as u32,
+
+            key: key.clone(),
+        };
+
+        slice.destroy().await;
+
+        self.indices.insert(index);
+
+        for listener in self.event_listeners.iter() {
+            listener.on_insert(key).await?;
+        }
+
+        let duration = now.elapsed() + writer.duration;
+        self.metrics
+            .latency_insert_admitted
+            .observe(duration.as_secs_f64());
+
+        Ok(true)
+    }
+}
+
+pub struct StoreWriter<'a, K, V, BA, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    store: &'a Store<K, V, BA, D, EP, EL>,
+    key: K,
+    weight: usize,
+
+    admitted: Option<bool>,
+
+    /// judge duration
+    duration: Duration,
+
+    applied: bool,
+}
+
+impl<'a, K, V, BA, D, EP, EL> StoreWriter<'a, K, V, BA, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    fn new(store: &'a Store<K, V, BA, D, EP, EL>, key: K, weight: usize) -> Self {
+        Self {
+            store,
+            key,
+            weight,
+            admitted: None,
+            duration: Duration::from_nanos(0),
+            applied: false,
+        }
+    }
+
+    /// Judge if the entry can be admitted by configured admission policies.
+    pub async fn judge(&mut self) -> bool {
+        let now = Instant::now();
+        if let Some(admitted) = self.admitted {
+            self.duration += now.elapsed();
+            return admitted;
+        }
+        let admitted = self.store.judge_inner(self).await;
+        self.admitted = Some(admitted);
+        self.duration += now.elapsed();
+        admitted
+    }
+
+    pub async fn finish(mut self, value: V) -> Result<bool> {
+        self.applied = true;
+        self.store.apply_writer(self, value).await
+    }
+}
+
+impl<'a, K, V, BA, D, EP, EL> Debug for StoreWriter<'a, K, V, BA, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("StoreWriter")
+            .field("key", &self.key)
+            .field("weight", &self.weight)
+            .field("admitted", &self.admitted)
+            .field("duration", &self.duration)
+            .field("applied", &self.applied)
+            .finish()
+    }
+}
+
+impl<'a, K, V, BA, D, EP, EL> Drop for StoreWriter<'a, K, V, BA, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    BA: BufferAllocator,
+    D: Device<IoBufferAllocator = BA>,
+    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    EL: Link,
+{
+    fn drop(&mut self) {
+        if !self.applied {
+            self.store
+                .metrics
+                .latency_insert_rejected
+                .observe(self.duration.as_secs_f64());
+        }
+    }
 }
 
 #[derive(Debug)]

From f7953bed19ff57e5d486eb6992a567433cae6ef5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 18 Jul 2023 01:49:10 +0800
Subject: [PATCH 055/261] fix: writer weight calculate (#76)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 535ee6d4..713a58de 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -248,7 +248,7 @@ where
 
     #[tracing::instrument(skip(self))]
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
-        let weight = self.serialized_len(&key, &value);
+        let weight = key.serialized_len() + value.serialized_len();
         let writer = StoreWriter::new(self, key, weight);
         writer.finish(value).await
     }
@@ -460,7 +460,7 @@ where
         }
 
         let serialized_len = self.serialized_len(key, &value);
-        assert_eq!(serialized_len, writer.weight);
+        assert_eq!(key.serialized_len() + value.serialized_len(), writer.weight);
 
         self.metrics.bytes_insert.inc_by(serialized_len as u64);
 

From 9f523b3aa865df919f31bf66d1ce46ebd98062a5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 20 Jul 2023 17:35:49 +0800
Subject: [PATCH 056/261] feat: refactor admission policy interface, fix rated
 random with multiple policies (#77)

* feat: refactor admission interfaces

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix drop

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix writer inserted or dropped

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/utils.rs            |   2 +-
 foyer-storage/Cargo.toml                    |   1 +
 foyer-storage/src/admission/mod.rs          |  15 +-
 foyer-storage/src/admission/rated_random.rs | 204 ++++++++++++++------
 foyer-storage/src/device/fs.rs              |   2 +-
 foyer-storage/src/metrics.rs                |  12 +-
 foyer-storage/src/store.rs                  | 126 +++++++++---
 7 files changed, 259 insertions(+), 103 deletions(-)

diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
index 1344c5b1..630101c0 100644
--- a/foyer-storage-bench/src/utils.rs
+++ b/foyer-storage-bench/src/utils.rs
@@ -31,7 +31,7 @@ use std::path::{Path, PathBuf};
 use itertools::Itertools;
 use nix::{fcntl::readlink, sys::stat::stat};
 
-#[allow(unused_variables)]
+#[allow(dead_code)]
 #[derive(PartialEq, Clone, Copy, Debug)]
 pub enum FsType {
     Xfs,
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index d3006846..e6cacbf4 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -11,6 +11,7 @@ license = "Apache-2.0"
 async-channel = "1.8"
 async-trait = "0.1"
 bitflags = "2.3.1"
+bitmaps = "3.2"
 bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 0be895b7..57dab869 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -12,20 +12,25 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use async_trait::async_trait;
+use bitmaps::Bitmap;
 use foyer_common::code::{Key, Value};
 
-use std::fmt::Debug;
+use std::{fmt::Debug, sync::Arc};
+
+use crate::metrics::Metrics;
+
+pub type Judges = Bitmap<64>;
 
 #[allow(unused_variables)]
-#[async_trait]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    async fn judge(&self, key: &Self::Key, weight: usize) -> bool;
+    fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
+
+    fn on_insert(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
 
-    async fn admit(&self, key: &Self::Key, weight: usize);
+    fn on_drop(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
 }
 
 pub mod rated_random;
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
index 7bb714d5..adba2e69 100644
--- a/foyer-storage/src/admission/rated_random.rs
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -15,31 +15,49 @@
 use std::{
     fmt::Debug,
     marker::PhantomData,
-    sync::atomic::{AtomicUsize, Ordering},
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
     time::{Duration, Instant},
 };
 
-use async_trait::async_trait;
 use foyer_common::code::{Key, Value};
 use parking_lot::{Mutex, MutexGuard};
 use rand::{thread_rng, Rng};
 
+use crate::metrics::Metrics;
+
 use super::AdmissionPolicy;
 
 const PRECISION: usize = 100000;
 
 /// p       : admit probability among △t
-/// w(t)    : weight at time t
+/// w_t    :  entry weight at time t
 /// r       : actual admitted rate
 /// E(w)    : expected admitted weight
 /// E(r)    : expceted admitted rate
 ///
-/// E(w(t)) = p * w(t)
-/// r = sum(△t){ admitted w(t) } / △t
-/// E(r) = sum(△t){ E(w(t)) } / △t
-///      = sum(△t){ p * w(t) } / △t
-///      = p / △t * sum(△t){ w(t) }
-/// p = E(r) * △t / sum(△t){ w(t) }
+/// E(w_t) = {
+///   p * w  ( if  judge &&  insert )
+///   p * w  ( if !judge && !insert )
+///   w      ( if !judge &&  insert )
+///   0      ( if  judge && !insert )
+/// }
+///
+///    E(r) = sum_{△t}{E(w_t)} / △t
+/// => E(r) * △t = sum_{△t}{ E(w_t) }
+/// => E(r) * △t =   sum_{△t}^{(judge && insert) || (!judge && !insert)}{ p * w_t }
+///                + sum_{△t}^{(!judge && insert)}{ w_t }
+///                + sum_{△t}^{(judge && !insert){ 0 }
+/// => E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t } = p * sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t }
+/// => p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
+///
+/// p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
+///     ↑ rate        ↑ △force_insert_bytes                   ↑ △obey_bytes
+///
+/// p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
+#[derive(Debug)]
 pub struct RatedRandom<K, V>
 where
     K: Key,
@@ -48,34 +66,21 @@ where
     rate: usize,
     update_interval: Duration,
 
-    bytes: AtomicUsize,
+    obey_bytes: AtomicUsize,
+    force_inser_bytes: AtomicUsize,
+
     probability: AtomicUsize,
 
-    inner: Mutex<Inner<K, V>>,
-}
+    inner: Mutex<Inner>,
 
-impl<K, V> Debug for RatedRandom<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("DynamicRandom")
-            .field("rate", &self.rate)
-            .field("probability", &self.probability.load(Ordering::Relaxed))
-            .finish()
-    }
+    _marker: PhantomData<(K, V)>,
 }
 
-struct Inner<K, V>
-where
-    K: Key,
-    V: Value,
-{
+#[derive(Debug)]
+struct Inner {
     last_update_time: Option<Instant>,
-    last_bytes: usize,
-
-    _marker: PhantomData<(K, V)>,
+    last_obey_bytes: usize,
+    last_force_insert_bytes: usize,
 }
 
 impl<K, V> RatedRandom<K, V>
@@ -87,29 +92,48 @@ where
         Self {
             rate,
             update_interval,
-            bytes: AtomicUsize::new(0),
+
+            obey_bytes: AtomicUsize::new(0),
+            force_inser_bytes: AtomicUsize::new(0),
+
             probability: AtomicUsize::new(0),
             inner: Mutex::new(Inner {
                 last_update_time: None,
-                last_bytes: 0,
-                _marker: PhantomData,
+                last_obey_bytes: 0,
+                last_force_insert_bytes: 0,
             }),
+            _marker: PhantomData,
         }
     }
 
-    fn judge(&self, _key: &K, weight: usize) -> bool {
+    fn judge(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
         if let Some(inner) = self.inner.try_lock() {
             self.update(inner);
         }
 
-        self.bytes.fetch_add(weight, Ordering::Relaxed);
-
         thread_rng().gen_range(0..PRECISION) < self.probability.load(Ordering::Relaxed)
     }
 
-    fn admit(&self, _key: &K, _weight: usize) {}
+    fn on_insert(&self, _key: &K, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        if judge {
+            // obey
+            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
+        } else {
+            // force insert
+            self.force_inser_bytes.fetch_add(weight, Ordering::Relaxed);
+        }
+    }
+
+    fn on_drop(&self, _key: &K, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        if !judge {
+            // obey
+            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
+        }
+    }
+
+    fn update(&self, mut inner: MutexGuard<'_, Inner>) {
+        // p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
 
-    fn update(&self, mut inner: MutexGuard<'_, Inner<K, V>>) {
         let now = Instant::now();
 
         let elapsed = match inner.last_update_time {
@@ -121,19 +145,35 @@ where
             return;
         }
 
-        let bytes = self.bytes.load(Ordering::Relaxed);
-        let bytes_delta = std::cmp::max(bytes - inner.last_bytes, 1);
+        let now_obey_bytes = self.obey_bytes.load(Ordering::Relaxed);
+        let now_force_insert_bytes = self.force_inser_bytes.load(Ordering::Relaxed);
+
+        let delta_obey_bytes = now_obey_bytes - inner.last_obey_bytes;
+        let delta_force_insert_bytes = now_force_insert_bytes - inner.last_force_insert_bytes;
+
+        inner.last_update_time = Some(now);
+        inner.last_obey_bytes = now_obey_bytes;
+        inner.last_force_insert_bytes = now_force_insert_bytes;
+
+        let p = if delta_obey_bytes == 0 {
+            1.0
+        } else {
+            let numerator =
+                self.rate as f64 * elapsed.as_secs_f64() - delta_force_insert_bytes as f64;
+            let numerator = numerator.abs();
+            let p = numerator / delta_obey_bytes as f64;
+            p.min(1.0)
+        };
+
+        debug_assert!((0.0..=1.0).contains(&p), "p out of range 0..=1: {}", p);
 
-        let p = self.rate as f64 * elapsed.as_secs_f64() / bytes_delta as f64;
         let p = (p * PRECISION as f64) as usize;
         self.probability.store(p, Ordering::Relaxed);
 
-        inner.last_update_time = Some(now);
-        inner.last_bytes = bytes;
+        tracing::debug!("probability: {}", p as f64 / PRECISION as f64);
     }
 }
 
-#[async_trait]
 impl<K, V> AdmissionPolicy for RatedRandom<K, V>
 where
     K: Key,
@@ -143,12 +183,16 @@ where
 
     type Value = V;
 
-    async fn judge(&self, key: &Self::Key, weight: usize) -> bool {
-        self.judge(key, weight)
+    fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool {
+        self.judge(key, weight, metrics)
+    }
+
+    fn on_insert(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool) {
+        self.on_insert(key, weight, metrics, judge)
     }
 
-    async fn admit(&self, key: &Self::Key, weight: usize) {
-        self.admit(key, weight)
+    fn on_drop(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool) {
+        self.on_drop(key, weight, metrics, judge)
     }
 }
 
@@ -156,14 +200,39 @@ where
 mod tests {
     use std::sync::Arc;
 
+    use itertools::Itertools;
+
     use super::*;
 
     #[ignore]
     #[tokio::test]
     async fn test_rated_random() {
-        const RATE: usize = 1_000_000;
+        const CASES: usize = 10;
         const ERATIO: f64 = 0.1;
 
+        let handles = (0..CASES).map(|_| tokio::spawn(case())).collect_vec();
+        let mut eratios = vec![];
+        for handle in handles {
+            let eratio = handle.await.unwrap();
+            assert!(eratio < ERATIO, "eratio: {} < ERATIO: {}", eratio, ERATIO);
+            eratios.push(eratio);
+        }
+        println!("========== RatedRandom error ratio begin ==========");
+        for eratio in eratios {
+            println!("eratio: {eratio}");
+        }
+        println!("=========== RatedRandom error ratio end ===========");
+    }
+
+    async fn case() -> f64 {
+        const RATE: usize = 1_000_000;
+        const CONCURRENCY: usize = 10;
+
+        const P_OTHER: f64 = 0.8;
+        const P_FORCE: f64 = 0.1;
+
+        let metrics = Arc::new(Metrics::default());
+
         let score = Arc::new(AtomicUsize::new(0));
 
         let rr = Arc::new(RatedRandom::<u64, Vec<u8>>::new(
@@ -171,26 +240,41 @@ mod tests {
             Duration::from_millis(100),
         ));
 
-        async fn submit(rr: Arc<RatedRandom<u64, Vec<u8>>>, score: Arc<AtomicUsize>) {
+        // scope: CONCURRENCY * (1 / interval) * range
+        // [1_000_000, 10_000_000]
+        // FORCE: [100_000, 1_000_000]
+
+        async fn submit(
+            rr: Arc<RatedRandom<u64, Vec<u8>>>,
+            score: Arc<AtomicUsize>,
+            metrics: Arc<Metrics>,
+        ) {
             loop {
-                tokio::time::sleep(Duration::from_millis(10)).await;
-                let weight = thread_rng().gen_range(1000..10000);
-                if rr.judge(&0, weight) {
+                tokio::time::sleep(Duration::from_millis(1)).await;
+                let weight = thread_rng().gen_range(100..1000);
+
+                let judge = rr.judge(&0, weight, &metrics);
+                let p_other = thread_rng().gen_range(0.0..=1.0);
+                let p_force = thread_rng().gen_range(0.0..=1.0);
+
+                let insert = (p_force <= P_FORCE) || (p_other <= P_OTHER && judge);
+
+                if insert {
                     score.fetch_add(weight, Ordering::Relaxed);
-                    rr.admit(&0, weight);
+                    rr.on_insert(&0, weight, &metrics, judge);
+                } else {
+                    rr.on_drop(&0, weight, &metrics, judge);
                 }
             }
         }
 
-        for _ in 0..10 {
-            tokio::spawn(submit(rr.clone(), score.clone()));
+        for _ in 0..CONCURRENCY {
+            tokio::spawn(submit(rr.clone(), score.clone(), metrics.clone()));
         }
 
         tokio::time::sleep(Duration::from_secs(10)).await;
         let s = score.load(Ordering::Relaxed);
         let error = (s as isize - RATE as isize * 10).unsigned_abs();
-        let eratio = error as f64 / (RATE as f64 * 10.0);
-
-        assert!(eratio < ERATIO, "eratio: {} < ERATIO: {}", eratio, ERATIO);
+        error as f64 / (RATE as f64 * 10.0)
     }
 }
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 15f29ede..08968233 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -61,7 +61,7 @@ impl FsDeviceConfig {
 struct FsDeviceInner {
     config: FsDeviceConfig,
 
-    #[allow(unused_variables)]
+    #[allow(dead_code)]
     dir: File,
 
     files: Vec<File>,
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index ad36fef2..8e9404a6 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -20,8 +20,8 @@ use prometheus::{
 
 #[derive(Debug)]
 pub struct Metrics {
-    pub latency_insert_admitted: Histogram,
-    pub latency_insert_rejected: Histogram,
+    pub latency_insert_inserted: Histogram,
+    pub latency_insert_dropped: Histogram,
     pub latency_lookup_hit: Histogram,
     pub latency_lookup_miss: Histogram,
     pub latency_remove: Histogram,
@@ -69,8 +69,8 @@ impl Metrics {
             register_int_counter_vec_with_registry!(opts, &["op", "extra"], registry).unwrap()
         };
 
-        let latency_insert_admitted = latency.with_label_values(&["insert", "admitted"]);
-        let latency_insert_rejected = latency.with_label_values(&["insert", "rejected"]);
+        let latency_insert_inserted = latency.with_label_values(&["insert", "inserted"]);
+        let latency_insert_dropped = latency.with_label_values(&["insert", "dropped"]);
         let latency_lookup_hit = latency.with_label_values(&["lookup", "hit"]);
         let latency_lookup_miss = latency.with_label_values(&["lookup", "miss"]);
         let latency_remove = latency.with_label_values(&["remove", ""]);
@@ -88,8 +88,8 @@ impl Metrics {
         };
 
         Self {
-            latency_insert_admitted,
-            latency_insert_rejected,
+            latency_insert_inserted,
+            latency_insert_dropped,
             latency_lookup_hit,
             latency_lookup_miss,
             latency_remove,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 713a58de..96321d0b 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -15,6 +15,7 @@
 use std::{
     fmt::Debug,
     marker::PhantomData,
+    ops::{BitAnd, BitOr},
     sync::Arc,
     time::{Duration, Instant},
 };
@@ -28,7 +29,7 @@ use tokio::{sync::broadcast, task::JoinHandle};
 use twox_hash::XxHash64;
 
 use crate::{
-    admission::AdmissionPolicy,
+    admission::{AdmissionPolicy, Judges},
     device::{BufferAllocator, Device},
     error::{Error, Result},
     event::EventListener,
@@ -427,13 +428,13 @@ where
         Ok(res)
     }
 
-    async fn judge_inner(&self, writer: &StoreWriter<'_, K, V, BA, D, EP, EL>) -> bool {
-        for admission in &self.admissions {
-            if !admission.judge(&writer.key, writer.weight).await {
-                return false;
-            }
+    fn judge_inner(&self, writer: &StoreWriter<'_, K, V, BA, D, EP, EL>) -> Judges {
+        let mut res = Judges::new();
+        for (index, admission) in self.admissions.iter().enumerate() {
+            let admitted = admission.judge(&writer.key, writer.weight, &self.metrics);
+            res.set(index, admitted);
         }
-        true
+        res
     }
 
     async fn apply_writer(
@@ -441,22 +442,24 @@ where
         mut writer: StoreWriter<'_, K, V, BA, D, EP, EL>,
         value: V,
     ) -> Result<bool> {
+        debug_assert!(!writer.inserted);
+
         let now = Instant::now();
 
-        if !writer.judge().await {
+        if !writer.judge() {
             let duration = now.elapsed() + writer.duration;
             self.metrics
-                .latency_insert_rejected
+                .latency_insert_dropped
                 .observe(duration.as_secs_f64());
             return Ok(false);
         }
 
-        assert!(writer.admitted.unwrap());
-
+        writer.inserted = true;
         let key = &writer.key;
 
-        for admission in &self.admissions {
-            admission.admit(key, writer.weight).await;
+        for (i, admission) in self.admissions.iter().enumerate() {
+            let judge = writer.judges.as_ref().unwrap().get(i);
+            admission.on_insert(key, writer.weight, &self.metrics, judge);
         }
 
         let serialized_len = self.serialized_len(key, &value);
@@ -501,7 +504,7 @@ where
 
         let duration = now.elapsed() + writer.duration;
         self.metrics
-            .latency_insert_admitted
+            .latency_insert_inserted
             .observe(duration.as_secs_f64());
 
         Ok(true)
@@ -521,12 +524,17 @@ where
     key: K,
     weight: usize,
 
-    admitted: Option<bool>,
+    /// 1: admit
+    /// 0: reject
+    judges: Option<Judges>,
+    /// 1: use
+    /// 0: ignore
+    mask: Judges,
 
     /// judge duration
     duration: Duration,
 
-    applied: bool,
+    inserted: bool,
 }
 
 impl<'a, K, V, BA, D, EP, EL> StoreWriter<'a, K, V, BA, D, EP, EL>
@@ -543,27 +551,52 @@ where
             store,
             key,
             weight,
-            admitted: None,
+            judges: None,
+            mask: Judges::from_value((1 << store.admissions.len()) - 1),
             duration: Duration::from_nanos(0),
-            applied: false,
+            inserted: false,
         }
     }
 
+    pub fn set_admission_mask(&mut self, mask: Judges) {
+        self.mask = mask.bitand(Judges::from_value((1 << self.store.admissions.len()) - 1));
+    }
+
+    pub fn all_admission(&mut self) {
+        self.mask = Judges::from_value((1 << self.store.admissions.len()) - 1);
+    }
+
+    pub fn ignore_admission(&mut self) {
+        self.mask = Judges::new();
+    }
+
     /// Judge if the entry can be admitted by configured admission policies.
-    pub async fn judge(&mut self) -> bool {
+    pub fn judge(&mut self) -> bool {
         let now = Instant::now();
-        if let Some(admitted) = self.admitted {
+        if let Some(judges) = self.judges {
             self.duration += now.elapsed();
-            return admitted;
+            return Self::is_admitted(&judges, &self.mask);
         }
-        let admitted = self.store.judge_inner(self).await;
-        self.admitted = Some(admitted);
+        let judges = self.store.judge_inner(self);
+        self.judges = Some(judges);
         self.duration += now.elapsed();
-        admitted
+        Self::is_admitted(&judges, &self.mask)
+    }
+
+    /// judge | ( ~mask )
+    ///
+    /// | judge | mask | ~mask | result |
+    /// |   0   |  0   |   1   |    1   |
+    /// |   0   |  1   |   0   |    0   |
+    /// |   1   |  0   |   1   |    1   |
+    /// |   1   |  1   |   0   |    1   |
+    fn is_admitted(judges: &Judges, mask: &Judges) -> bool {
+        let mut umask = *mask;
+        umask.invert();
+        judges.bitor(umask).is_full()
     }
 
-    pub async fn finish(mut self, value: V) -> Result<bool> {
-        self.applied = true;
+    pub async fn finish(self, value: V) -> Result<bool> {
         self.store.apply_writer(self, value).await
     }
 }
@@ -581,9 +614,9 @@ where
         f.debug_struct("StoreWriter")
             .field("key", &self.key)
             .field("weight", &self.weight)
-            .field("admitted", &self.admitted)
+            .field("judged", &self.judges)
             .field("duration", &self.duration)
-            .field("applied", &self.applied)
+            .field("inserted", &self.inserted)
             .finish()
     }
 }
@@ -598,11 +631,17 @@ where
     EL: Link,
 {
     fn drop(&mut self) {
-        if !self.applied {
+        if !self.inserted {
             self.store
                 .metrics
-                .latency_insert_rejected
+                .latency_insert_dropped
                 .observe(self.duration.as_secs_f64());
+            if let Some(judge) = self.judges.as_ref() {
+                for (i, admission) in self.store.admissions.iter().enumerate() {
+                    let judge = judge.get(i);
+                    admission.on_drop(&self.key, self.weight, &self.store.metrics, judge);
+                }
+            }
         }
     }
 }
@@ -968,4 +1007,31 @@ pub mod tests {
         store.shutdown_runners().await.unwrap();
         drop(store);
     }
+
+    #[test]
+    fn test_admission_mask() {
+        type T = StoreWriter<
+            'static,
+            u64,
+            Vec<u8>,
+            AlignedAllocator,
+            FsDevice,
+            Fifo<RegionEpItemAdapter<FifoLink>>,
+            FifoLink,
+        >;
+
+        let mask = Judges::from_value(0b_0011);
+
+        assert!(mask.get(0));
+        assert!(mask.get(1));
+        assert!(!mask.get(2));
+        assert!(!mask.get(3));
+
+        let j1 = Judges::from_value(0b_0011);
+        let j2 = Judges::from_value(0b_1011);
+        let j3 = Judges::from_value(0b_1010);
+        assert!(T::is_admitted(&j1, &mask));
+        assert!(T::is_admitted(&j2, &mask));
+        assert!(!T::is_admitted(&j3, &mask));
+    }
 }

From 67c99d2b69296b509c4e1051b99ee287646bd3aa Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 21 Jul 2023 15:18:13 +0800
Subject: [PATCH 057/261] feat: add more insert interfaces (#78)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 75 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 96321d0b..35451f67 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -16,6 +16,7 @@ use std::{
     fmt::Debug,
     marker::PhantomData,
     ops::{BitAnd, BitOr},
+    pin::Pin,
     sync::Arc,
     time::{Duration, Instant},
 };
@@ -23,6 +24,7 @@ use std::{
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, queue::AsyncQueue, rate::RateLimiter};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use futures::Future;
 use itertools::Itertools;
 use parking_lot::Mutex;
 use tokio::{sync::broadcast, task::JoinHandle};
@@ -254,6 +256,79 @@ where
         writer.finish(value).await
     }
 
+    #[tracing::instrument(skip(self))]
+    pub async fn insert_if_not_exists(&self, key: K, value: V) -> Result<bool> {
+        if !self.exists(&key)? {
+            return Ok(false);
+        }
+        self.insert(key, value).await
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    pub async fn insert_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
+    where
+        F: Fn(&K) -> V,
+    {
+        let mut writer = self.writer(key, weight);
+        if !writer.judge() {
+            return Ok(false);
+        }
+        let value = f(&writer.key);
+        writer.finish(value).await
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called to fetch value, and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    pub async fn insert_with_future<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
+    where
+        F: Fn(&K) -> Pin<Box<dyn Future<Output = V>>>,
+    {
+        let mut writer = self.writer(key, weight);
+        if !writer.judge() {
+            return Ok(false);
+        }
+        let value = f(&writer.key).await;
+        writer.finish(value).await
+    }
+
+    #[tracing::instrument(skip(self, f))]
+    pub async fn insert_if_not_exists_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
+    where
+        F: Fn(&K) -> V,
+    {
+        if !self.exists(&key)? {
+            return Ok(false);
+        }
+        self.insert_with(key, f, weight).await
+    }
+
+    #[tracing::instrument(skip(self, f))]
+    pub async fn insert_if_not_exists_with_future<F>(
+        &self,
+        key: K,
+        f: F,
+        weight: usize,
+    ) -> Result<bool>
+    where
+        F: Fn(&K) -> Pin<Box<dyn Future<Output = V>>>,
+    {
+        if !self.exists(&key)? {
+            return Ok(false);
+        }
+        self.insert_with_future(key, f, weight).await
+    }
+
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[tracing::instrument(skip(self))]
     pub fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, BA, D, EP, EL> {

From a8c1c78797c1d982be8d304b4476a856010bdaa7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 21 Jul 2023 16:33:55 +0800
Subject: [PATCH 058/261] refactor: insert with interface allow return fetch
 value error (#79)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/Cargo.toml   |  1 +
 foyer-storage/src/error.rs | 18 ++++++++++++------
 foyer-storage/src/store.rs | 12 ++++++------
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index e6cacbf4..baa13504 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1.0"
 async-channel = "1.8"
 async-trait = "0.1"
 bitflags = "2.3.1"
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index 1821bb4b..24996b5d 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -23,9 +23,11 @@ pub enum Error {
     #[error("channel full")]
     ChannelFull,
     #[error("event listener error: {0}")]
-    EventListener(#[from] Box<dyn std::error::Error + Send + Sync + 'static>),
+    EventListener(Box<dyn std::error::Error + Send + Sync + 'static>),
+    #[error("fetch value error: {0}")]
+    FetchValue(anyhow::Error),
     #[error("other error: {0}")]
-    Other(String),
+    Other(#[from] anyhow::Error),
 }
 
 impl Error {
@@ -33,15 +35,19 @@ impl Error {
         Self::Device(e)
     }
 
+    pub fn fetch_value(e: impl Into<anyhow::Error>) -> Self {
+        Self::Other(e.into())
+    }
+
+    pub fn other(e: impl Into<anyhow::Error>) -> Self {
+        Self::Other(e.into())
+    }
+
     pub fn event_listener(
         e: impl Into<Box<dyn std::error::Error + Send + Sync + 'static>>,
     ) -> Self {
         Self::EventListener(e.into())
     }
-
-    pub fn other(e: impl Into<Box<dyn std::error::Error>>) -> Self {
-        Self::Other(e.into().to_string())
-    }
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 35451f67..9e92a528 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -273,13 +273,13 @@ where
     #[tracing::instrument(skip(self, f))]
     pub async fn insert_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
     where
-        F: Fn(&K) -> V,
+        F: Fn(&K) -> anyhow::Result<V>,
     {
         let mut writer = self.writer(key, weight);
         if !writer.judge() {
             return Ok(false);
         }
-        let value = f(&writer.key);
+        let value = f(&writer.key).map_err(Error::fetch_value)?;
         writer.finish(value).await
     }
 
@@ -292,20 +292,20 @@ where
     #[tracing::instrument(skip(self, f))]
     pub async fn insert_with_future<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
     where
-        F: Fn(&K) -> Pin<Box<dyn Future<Output = V>>>,
+        F: Fn(&K) -> Pin<Box<dyn Future<Output = anyhow::Result<V>>>>,
     {
         let mut writer = self.writer(key, weight);
         if !writer.judge() {
             return Ok(false);
         }
-        let value = f(&writer.key).await;
+        let value = f(&writer.key).await.map_err(Error::fetch_value)?;
         writer.finish(value).await
     }
 
     #[tracing::instrument(skip(self, f))]
     pub async fn insert_if_not_exists_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
     where
-        F: Fn(&K) -> V,
+        F: Fn(&K) -> anyhow::Result<V>,
     {
         if !self.exists(&key)? {
             return Ok(false);
@@ -321,7 +321,7 @@ where
         weight: usize,
     ) -> Result<bool>
     where
-        F: Fn(&K) -> Pin<Box<dyn Future<Output = V>>>,
+        F: Fn(&K) -> Pin<Box<dyn Future<Output = anyhow::Result<V>>>>,
     {
         if !self.exists(&key)? {
             return Ok(false);

From 21e6acecf051d3155b5cd301bf94d957abe03b9e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 24 Jul 2023 15:26:55 +0800
Subject: [PATCH 059/261] fix: refactor future interface in insert with (#80)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/metrics.rs |  3 +++
 foyer-storage/src/store.rs   | 39 ++++++++++++++++++++++++------------
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 8e9404a6..e1c83ae4 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -21,6 +21,7 @@ use prometheus::{
 #[derive(Debug)]
 pub struct Metrics {
     pub latency_insert_inserted: Histogram,
+    pub latency_insert_filtered: Histogram,
     pub latency_insert_dropped: Histogram,
     pub latency_lookup_hit: Histogram,
     pub latency_lookup_miss: Histogram,
@@ -70,6 +71,7 @@ impl Metrics {
         };
 
         let latency_insert_inserted = latency.with_label_values(&["insert", "inserted"]);
+        let latency_insert_filtered = latency.with_label_values(&["insert", "filtered"]);
         let latency_insert_dropped = latency.with_label_values(&["insert", "dropped"]);
         let latency_lookup_hit = latency.with_label_values(&["lookup", "hit"]);
         let latency_lookup_miss = latency.with_label_values(&["lookup", "miss"]);
@@ -89,6 +91,7 @@ impl Metrics {
 
         Self {
             latency_insert_inserted,
+            latency_insert_filtered,
             latency_insert_dropped,
             latency_lookup_hit,
             latency_lookup_miss,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 9e92a528..ccce3847 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -16,7 +16,6 @@ use std::{
     fmt::Debug,
     marker::PhantomData,
     ops::{BitAnd, BitOr},
-    pin::Pin,
     sync::Arc,
     time::{Duration, Instant},
 };
@@ -48,6 +47,8 @@ use std::hash::Hasher;
 
 const REGION_MAGIC: u64 = 0x19970327;
 
+pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
+
 #[derive(Debug, Default)]
 pub struct PrometheusConfig {
     pub registry: Option<prometheus::Registry>,
@@ -273,13 +274,13 @@ where
     #[tracing::instrument(skip(self, f))]
     pub async fn insert_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
     where
-        F: Fn(&K) -> anyhow::Result<V>,
+        F: FnOnce() -> anyhow::Result<V>,
     {
         let mut writer = self.writer(key, weight);
         if !writer.judge() {
             return Ok(false);
         }
-        let value = f(&writer.key).map_err(Error::fetch_value)?;
+        let value = f().map_err(Error::fetch_value)?;
         writer.finish(value).await
     }
 
@@ -290,22 +291,23 @@ where
     ///
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[tracing::instrument(skip(self, f))]
-    pub async fn insert_with_future<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
+    pub async fn insert_with_future<F, FU>(&self, key: K, f: F, weight: usize) -> Result<bool>
     where
-        F: Fn(&K) -> Pin<Box<dyn Future<Output = anyhow::Result<V>>>>,
+        F: FnOnce() -> FU,
+        FU: FetchValueFuture<V>,
     {
         let mut writer = self.writer(key, weight);
         if !writer.judge() {
             return Ok(false);
         }
-        let value = f(&writer.key).await.map_err(Error::fetch_value)?;
+        let value = f().await.map_err(Error::fetch_value)?;
         writer.finish(value).await
     }
 
     #[tracing::instrument(skip(self, f))]
     pub async fn insert_if_not_exists_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
     where
-        F: Fn(&K) -> anyhow::Result<V>,
+        F: FnOnce() -> anyhow::Result<V>,
     {
         if !self.exists(&key)? {
             return Ok(false);
@@ -314,14 +316,15 @@ where
     }
 
     #[tracing::instrument(skip(self, f))]
-    pub async fn insert_if_not_exists_with_future<F>(
+    pub async fn insert_if_not_exists_with_future<F, FU>(
         &self,
         key: K,
         f: F,
         weight: usize,
     ) -> Result<bool>
     where
-        F: Fn(&K) -> Pin<Box<dyn Future<Output = anyhow::Result<V>>>>,
+        F: FnOnce() -> FU,
+        FU: FetchValueFuture<V>,
     {
         if !self.exists(&key)? {
             return Ok(false);
@@ -522,10 +525,7 @@ where
         let now = Instant::now();
 
         if !writer.judge() {
-            let duration = now.elapsed() + writer.duration;
-            self.metrics
-                .latency_insert_dropped
-                .observe(duration.as_secs_f64());
+            writer.duration += now.elapsed();
             return Ok(false);
         }
 
@@ -711,11 +711,24 @@ where
                 .metrics
                 .latency_insert_dropped
                 .observe(self.duration.as_secs_f64());
+            let mut filtered = false;
             if let Some(judge) = self.judges.as_ref() {
                 for (i, admission) in self.store.admissions.iter().enumerate() {
                     let judge = judge.get(i);
                     admission.on_drop(&self.key, self.weight, &self.store.metrics, judge);
                 }
+                filtered = !self.judge();
+            }
+            if filtered {
+                self.store
+                    .metrics
+                    .latency_insert_filtered
+                    .observe(self.duration.as_secs_f64());
+            } else {
+                self.store
+                    .metrics
+                    .latency_insert_dropped
+                    .observe(self.duration.as_secs_f64());
             }
         }
     }

From 9669cf9ae968b79c6fc55e1e01bd18c140148155 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 24 Jul 2023 21:57:14 +0800
Subject: [PATCH 060/261] fix: reverse condition of insert if not exists (#81)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index ccce3847..24d0198a 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -259,7 +259,7 @@ where
 
     #[tracing::instrument(skip(self))]
     pub async fn insert_if_not_exists(&self, key: K, value: V) -> Result<bool> {
-        if !self.exists(&key)? {
+        if self.exists(&key)? {
             return Ok(false);
         }
         self.insert(key, value).await
@@ -309,7 +309,7 @@ where
     where
         F: FnOnce() -> anyhow::Result<V>,
     {
-        if !self.exists(&key)? {
+        if self.exists(&key)? {
             return Ok(false);
         }
         self.insert_with(key, f, weight).await
@@ -326,7 +326,7 @@ where
         F: FnOnce() -> FU,
         FU: FetchValueFuture<V>,
     {
-        if !self.exists(&key)? {
+        if self.exists(&key)? {
             return Ok(false);
         }
         self.insert_with_future(key, f, weight).await

From 7017be863bc00e85a3af1a193273184c9b82e932 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 25 Jul 2023 11:13:27 +0800
Subject: [PATCH 061/261] fix: remove mistake cfg attr (#82)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/device/mod.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index bb45d4eb..044570cc 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -76,7 +76,6 @@ where
     F: FnOnce() -> error::Result<T> + Send + 'static,
     T: Send + 'static,
 {
-    #[cfg_attr(madsim, expect(deprecated))]
     match tokio::task::spawn_blocking(f).await {
         Ok(res) => res,
         Err(e) => Err(error::Error::Other(format!(

From 270dc283b3caff3fc04d559823e4e2f3c1ef1dc5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 25 Jul 2023 14:22:03 +0800
Subject: [PATCH 062/261] fix: make slice drop recoverable (#83)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region.rs | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index cfe549d3..6375cbbc 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -348,7 +348,7 @@ where
 
 // read & write slice
 
-#[pin_project::pin_project(PinnedDrop)]
+#[pin_project::pin_project(project = WriteSliceProj, PinnedDrop)]
 pub struct WriteSlice {
     slice: SliceMut,
     region_id: RegionId,
@@ -415,12 +415,25 @@ impl AsMut<[u8]> for WriteSlice {
 #[pin_project::pinned_drop]
 impl PinnedDrop for WriteSlice {
     fn drop(self: Pin<&mut Self>) {
-        if self.future.is_some() {
-            panic!("future is not consumed: {:?}", self);
+        let mut this = self.project();
+        if let Some(future) = this.future.take() {
+            tracing::error!("future is not consumed. This may be caused by error early return. If there's not, check if there's slice not destroyed. {:?}", this);
+            tokio::spawn(future);
         }
     }
 }
 
+impl<'pin> Debug for WriteSliceProj<'pin> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("WriteSlice")
+            .field("slice", &self.slice)
+            .field("region_id", &self.region_id)
+            .field("version", &self.version)
+            .field("offset", &self.offset)
+            .finish()
+    }
+}
+
 #[pin_project::pin_project(project = ReadSliceProj, PinnedDrop)]
 pub enum ReadSlice<A>
 where
@@ -448,11 +461,14 @@ where
             Self::Slice {
                 slice, allocator, ..
             } => f
-                .debug_struct("Slice")
+                .debug_struct("ReadSlice::Slice")
                 .field("slice", slice)
                 .field("allocator", allocator)
                 .finish(),
-            Self::Owned { buf, .. } => f.debug_struct("Owned").field("buf", buf).finish(),
+            Self::Owned { buf, .. } => f
+                .debug_struct("ReadSlice::Owned")
+                .field("buf", buf)
+                .finish(),
         }
     }
 }
@@ -504,11 +520,12 @@ where
 {
     fn drop(self: Pin<&mut Self>) {
         let mut this = self.project();
-        if match &mut this {
-            ReadSliceProj::Slice { future, .. } => future.is_some(),
-            ReadSliceProj::Owned { future, .. } => future.is_some(),
+        if let Some(future) = match &mut this {
+            ReadSliceProj::Slice { future, .. } => future.take(),
+            ReadSliceProj::Owned { future, .. } => future.take(),
         } {
-            panic!("future is not consumed: {:?}", this);
+            tracing::error!("future is not consumed. This may be caused by error early return. If there's not, check if there's slice not destroyed. {:?}", this);
+            tokio::spawn(future);
         }
     }
 }

From af1e04a45f9226fbca7b90f026a4d4569c25df01 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 25 Jul 2023 21:51:21 +0800
Subject: [PATCH 063/261] chore: add makefile for check and test (#87)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 Makefile

diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..4b05f67c
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,8 @@
+SHELL := /bin/bash
+.PHONY: proto check test
+
+check:
+	cargo sort -w && cargo fmt --all && cargo clippy --all-targets
+
+test:
+	cargo test --all
\ No newline at end of file

From 72731f7e2132d5792407e965a3b3b9d49ed00d66 Mon Sep 17 00:00:00 2001
From: William Wen <44139337+wenym1@users.noreply.github.com>
Date: Wed, 26 Jul 2023 12:07:25 +0800
Subject: [PATCH 064/261] refactor: simplify some generic type definition (#85)

* refactor: simplify some generic type definition

* run clippy
---
 foyer-intrusive/src/eviction/fifo.rs |  17 ++---
 foyer-intrusive/src/eviction/lfu.rs  |   8 +-
 foyer-intrusive/src/eviction/lru.rs  |  11 +--
 foyer-intrusive/src/eviction/mod.rs  |  45 ++++++-----
 foyer-intrusive/src/lib.rs           |   1 +
 foyer-memory/src/lib.rs              |  17 ++---
 foyer-storage/src/flusher.rs         |  29 ++++----
 foyer-storage/src/lib.rs             |   8 --
 foyer-storage/src/reclaimer.rs       |  31 ++++----
 foyer-storage/src/region.rs          |  20 +++--
 foyer-storage/src/region_manager.rs  |  34 ++++-----
 foyer-storage/src/store.rs           | 107 +++++++++++----------------
 12 files changed, 145 insertions(+), 183 deletions(-)

diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 8abc336c..c7158a37 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -315,48 +315,44 @@ where
 unsafe impl<A> Send for Fifo<A>
 where
     A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
-
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
+
 unsafe impl<A> Sync for Fifo<A>
 where
     A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
-
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 
 unsafe impl Send for FifoLink {}
+
 unsafe impl Sync for FifoLink {}
 
 unsafe impl<'a, A> Send for FifoIter<'a, A>
 where
     A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
-
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
+
 unsafe impl<'a, A> Sync for FifoIter<'a, A>
 where
     A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
-
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 
-impl<A> EvictionPolicy<A> for Fifo<A>
+impl<A> EvictionPolicy for Fifo<A>
 where
     A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
-
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
-    type Link = FifoLink;
+    type Adapter = A;
 
     type Config = FifoConfig;
 
-    type E<'e> = FifoIter<'e, A>;
-
     fn new(config: Self::Config) -> Self {
         Self::new(config)
     }
@@ -380,7 +376,7 @@ where
         self.len()
     }
 
-    fn iter(&self) -> Self::E<'_> {
+    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
         self.iter()
     }
 }
@@ -389,6 +385,7 @@ where
 mod tests {
     use std::sync::Arc;
 
+    use crate::eviction::EvictionPolicyExt;
     use itertools::Itertools;
 
     use crate::priority_adapter;
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 4beed1a7..f10dbb5f 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -523,17 +523,15 @@ where
 {
 }
 
-impl<A> EvictionPolicy<A> for Lfu<A>
+impl<A> EvictionPolicy for Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
-    type Link = LfuLink;
+    type Adapter = A;
 
     type Config = LfuConfig;
 
-    type E<'e> = LfuIter<'e, A>;
-
     fn new(config: Self::Config) -> Self {
         Self::new(config)
     }
@@ -557,7 +555,7 @@ where
         self.len()
     }
 
-    fn iter(&self) -> Self::E<'_> {
+    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
         self.iter()
     }
 }
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 2a00466e..1e53ebab 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -352,6 +352,7 @@ where
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
+
 unsafe impl<A> Sync for Lru<A>
 where
     A: Adapter<Link = LruLink>,
@@ -360,6 +361,7 @@ where
 }
 
 unsafe impl Send for LruLink {}
+
 unsafe impl Sync for LruLink {}
 
 unsafe impl<'a, A> Send for LruIter<'a, A>
@@ -368,6 +370,7 @@ where
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
+
 unsafe impl<'a, A> Sync for LruIter<'a, A>
 where
     A: Adapter<Link = LruLink>,
@@ -375,17 +378,15 @@ where
 {
 }
 
-impl<A> EvictionPolicy<A> for Lru<A>
+impl<A> EvictionPolicy for Lru<A>
 where
     A: Adapter<Link = LruLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
-    type Link = LruLink;
+    type Adapter = A;
 
     type Config = LruConfig;
 
-    type E<'e> = LruIter<'e, A>;
-
     fn new(config: Self::Config) -> Self {
         Self::new(config)
     }
@@ -409,7 +410,7 @@ where
         self.len()
     }
 
-    fn iter(&self) -> Self::E<'_> {
+    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
         self.iter()
     }
 }
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index 7889d625..c44152d0 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -12,34 +12,26 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::core::{
-    adapter::{Adapter, Link},
-    pointer::PointerOps,
-};
+use crate::core::{adapter::Adapter, pointer::PointerOps};
 
 use std::fmt::Debug;
 
 pub trait Config = Send + Sync + 'static + Debug + Clone;
 
-pub trait EvictionPolicy<A>: Send + Sync + 'static
-where
-    A: Adapter<Link = Self::Link>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
-{
-    type Link: Link;
+pub trait EvictionPolicy: Send + Sync + 'static {
+    type Adapter: Adapter;
     type Config: Config;
-    type E<'e>: Iterator<Item = &'e <A::PointerOps as PointerOps>::Pointer>;
 
     fn new(config: Self::Config) -> Self;
 
-    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer);
+    fn insert(&mut self, ptr: <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer);
 
     fn remove(
         &mut self,
-        ptr: &<A::PointerOps as PointerOps>::Pointer,
-    ) -> <A::PointerOps as PointerOps>::Pointer;
+        ptr: &<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer,
+    ) -> <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer;
 
-    fn access(&mut self, ptr: &<A::PointerOps as PointerOps>::Pointer);
+    fn access(&mut self, ptr: &<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer);
 
     fn len(&self) -> usize;
 
@@ -47,13 +39,28 @@ where
         self.len() == 0
     }
 
-    fn iter(&self) -> Self::E<'_>;
+    fn iter(
+        &self,
+    ) -> impl Iterator<Item = &'_ <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer> + '_;
+}
+
+pub trait EvictionPolicyExt: EvictionPolicy {
+    fn push(&mut self, ptr: <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer);
+
+    fn pop(&mut self) -> Option<<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer>;
 
-    fn push(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    fn peek(&self) -> Option<&<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer>;
+}
+
+impl<E: EvictionPolicy> EvictionPolicyExt for E
+where
+    <<E::Adapter as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    fn push(&mut self, ptr: <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer) {
         self.insert(ptr)
     }
 
-    fn pop(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+    fn pop(&mut self) -> Option<<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer> {
         let ptr = {
             let mut iter = self.iter();
             let ptr = iter.next();
@@ -62,7 +69,7 @@ where
         ptr.map(|ptr| self.remove(&ptr))
     }
 
-    fn peek(&self) -> Option<&<A::PointerOps as PointerOps>::Pointer> {
+    fn peek(&self) -> Option<&<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer> {
         self.iter().next()
     }
 }
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index af07e036..717ac423 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -15,6 +15,7 @@
 #![feature(associated_type_bounds)]
 #![feature(ptr_metadata)]
 #![feature(trait_alias)]
+#![feature(return_position_impl_trait_in_trait)]
 #![allow(clippy::new_without_default)]
 #![allow(clippy::wrong_self_convention)]
 #![allow(clippy::vtable_address_comparisons)]
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 9b3aab6a..fdaf832f 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -25,12 +25,9 @@ use parking_lot::Mutex;
 use std::hash::Hasher;
 use twox_hash::XxHash64;
 
-pub struct CacheConfig<K, V, E, EL>
+pub struct CacheConfig<E>
 where
-    K: Key,
-    V: Value,
-    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
-    EL: Link,
+    E: EvictionPolicy,
 {
     capacity: usize,
     shard_bits: usize,
@@ -42,7 +39,7 @@ pub struct Cache<K, V, E, EL>
 where
     K: Key,
     V: Value,
-    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
+    E: EvictionPolicy<Adapter = CacheItemEpAdapter<K, V, EL>>,
     EL: Link,
 {
     shards: Vec<Mutex<CacheShard<K, V, E, EL>>>,
@@ -52,7 +49,7 @@ struct CacheShard<K, V, E, EL>
 where
     K: Key,
     V: Value,
-    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
+    E: EvictionPolicy<Adapter = CacheItemEpAdapter<K, V, EL>>,
     EL: Link,
 {
     container: HashMap<K, V, CacheItemHmAdapter<K, V, EL>>,
@@ -115,10 +112,10 @@ impl<K, V, E, EL> Cache<K, V, E, EL>
 where
     K: Key,
     V: Value,
-    E: EvictionPolicy<CacheItemEpAdapter<K, V, EL>, Link = EL>,
+    E: EvictionPolicy<Adapter = CacheItemEpAdapter<K, V, EL>>,
     EL: Link,
 {
-    pub fn new(config: CacheConfig<K, V, E, EL>) -> Self {
+    pub fn new(config: CacheConfig<E>) -> Self {
         let mut shards = Vec::with_capacity(1 << config.shard_bits);
 
         let shard_capacity = config.capacity / (1 << config.shard_bits);
@@ -231,7 +228,7 @@ mod tests {
         }
     }
 
-    type FifoCacheConfig = CacheConfig<K, V, Fifo<CacheItemEpAdapter<K, V, FifoLink>>, FifoLink>;
+    type FifoCacheConfig = CacheConfig<Fifo<CacheItemEpAdapter<K, V, FifoLink>>>;
     type FifoCache = Cache<K, V, Fifo<CacheItemEpAdapter<K, V, FifoLink>>, FifoLink>;
 
     #[test]
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 1efeb4d1..e4771413 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -60,18 +60,17 @@ impl Flusher {
         }
     }
 
-    pub async fn run<A, D, E, EL>(
+    pub async fn run<D, E, EL>(
         &self,
-        buffers: Arc<AsyncQueue<Vec<u8, A>>>,
-        region_manager: Arc<RegionManager<A, D, E, EL>>,
+        buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
+        region_manager: Arc<RegionManager<D, E, EL>>,
         rate_limiter: Option<Arc<RateLimiter>>,
         stop_rxs: Vec<broadcast::Receiver<()>>,
         metrics: Arc<Metrics>,
     ) -> Vec<JoinHandle<()>>
     where
-        A: BufferAllocator,
-        D: Device<IoBufferAllocator = A>,
-        E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+        D: Device,
+        E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
         EL: Link,
     {
         let mut inner = self.inner.lock().await;
@@ -118,17 +117,16 @@ impl Flusher {
     }
 }
 
-struct Runner<A, D, E, EL>
+struct Runner<D, E, EL>
 where
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
-    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     task_rx: mpsc::UnboundedReceiver<FlushTask>,
-    buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+    buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
 
-    region_manager: Arc<RegionManager<A, D, E, EL>>,
+    region_manager: Arc<RegionManager<D, E, EL>>,
 
     rate_limiter: Option<Arc<RateLimiter>>,
 
@@ -137,11 +135,10 @@ where
     metrics: Arc<Metrics>,
 }
 
-impl<A, D, E, EL> Runner<A, D, E, EL>
+impl<D, E, EL> Runner<D, E, EL>
 where
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
-    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     async fn run(mut self) -> Result<()> {
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 883d794c..c2e5fb52 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -19,8 +19,6 @@
 #![feature(let_chains)]
 #![allow(clippy::type_complexity)]
 
-use device::allocator::AlignedAllocator;
-
 pub mod admission;
 pub mod device;
 pub mod error;
@@ -38,7 +36,6 @@ pub mod store;
 pub type LruFsStore<K, V> = store::Store<
     K,
     V,
-    AlignedAllocator,
     device::fs::FsDevice,
     foyer_intrusive::eviction::lru::Lru<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
@@ -53,13 +50,11 @@ pub type LruFsStoreConfig<K, V> = store::StoreConfig<
     foyer_intrusive::eviction::lru::Lru<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
     >,
-    foyer_intrusive::eviction::lru::LruLink,
 >;
 
 pub type LfuFsStore<K, V> = store::Store<
     K,
     V,
-    AlignedAllocator,
     device::fs::FsDevice,
     foyer_intrusive::eviction::lfu::Lfu<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
@@ -74,13 +69,11 @@ pub type LfuFsStoreConfig<K, V> = store::StoreConfig<
     foyer_intrusive::eviction::lfu::Lfu<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
     >,
-    foyer_intrusive::eviction::lfu::LfuLink,
 >;
 
 pub type FifoFsStore<K, V> = store::Store<
     K,
     V,
-    AlignedAllocator,
     device::fs::FsDevice,
     foyer_intrusive::eviction::fifo::Fifo<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
@@ -95,5 +88,4 @@ pub type FifoFsStoreConfig<K, V> = store::StoreConfig<
     foyer_intrusive::eviction::fifo::Fifo<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
     >,
-    foyer_intrusive::eviction::fifo::FifoLink,
 >;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 3440dcda..dd633f6e 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use crate::{
-    device::{BufferAllocator, Device},
+    device::Device,
     error::{Error, Result},
     event::EventListener,
     indices::Indices,
@@ -69,10 +69,10 @@ impl Reclaimer {
     }
 
     #[allow(clippy::too_many_arguments)]
-    pub async fn run<K, V, A, D, EP, EL>(
+    pub async fn run<K, V, D, EP, EL>(
         &self,
-        store: Arc<Store<K, V, A, D, EP, EL>>,
-        region_manager: Arc<RegionManager<A, D, EP, EL>>,
+        store: Arc<Store<K, V, D, EP, EL>>,
+        region_manager: Arc<RegionManager<D, EP, EL>>,
         clean_regions: Arc<AsyncQueue<RegionId>>,
         reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
         indices: Arc<Indices<K>>,
@@ -84,9 +84,8 @@ impl Reclaimer {
     where
         K: Key,
         V: Value,
-        A: BufferAllocator,
-        D: Device<IoBufferAllocator = A>,
-        EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+        D: Device,
+        EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
         EL: Link,
     {
         let mut inner = self.inner.lock().await;
@@ -140,19 +139,18 @@ impl Reclaimer {
     }
 }
 
-struct Runner<K, V, A, D, EP, EL>
+struct Runner<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     task_rx: mpsc::Receiver<ReclaimTask>,
 
-    _store: Arc<Store<K, V, A, D, EP, EL>>,
-    region_manager: Arc<RegionManager<A, D, EP, EL>>,
+    _store: Arc<Store<K, V, D, EP, EL>>,
+    region_manager: Arc<RegionManager<D, EP, EL>>,
     clean_regions: Arc<AsyncQueue<RegionId>>,
     _reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
     indices: Arc<Indices<K>>,
@@ -166,13 +164,12 @@ where
     metrics: Arc<Metrics>,
 }
 
-impl<K, V, A, D, EP, EL> Runner<K, V, A, D, EP, EL>
+impl<K, V, D, EP, EL> Runner<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     async fn run(mut self) -> Result<()> {
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 6375cbbc..c6bf62dd 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -61,14 +61,13 @@ where
 }
 
 #[derive(Debug, Clone)]
-pub struct Region<A, D>
+pub struct Region<D>
 where
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
+    D: Device,
 {
     id: RegionId,
 
-    inner: ErwLock<A>,
+    inner: ErwLock<D::IoBufferAllocator>,
 
     device: D,
 }
@@ -85,10 +84,9 @@ where
 ///                    step 2 detaches dirty buffer, must guarantee there is no buffer readers
 /// - reclaim        : happens after the region is evicted, must guarantee there is no writers, buffer readers or physical readers,
 ///                    *or in-flight writers or readers* (verify by version)
-impl<A, D> Region<A, D>
+impl<D> Region<D>
 where
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
+    D: Device,
 {
     pub fn new(id: RegionId, device: D) -> Self {
         let inner = RegionInner {
@@ -168,7 +166,7 @@ where
         &self,
         range: impl RangeBounds<usize>,
         version: Version,
-    ) -> Result<Option<ReadSlice<A>>> {
+    ) -> Result<Option<ReadSlice<D::IoBufferAllocator>>> {
         let start = match range.start_bound() {
             std::ops::Bound::Included(i) => *i,
             std::ops::Bound::Excluded(i) => *i + 1,
@@ -255,7 +253,7 @@ where
         }))
     }
 
-    pub async fn attach_buffer(&self, buf: Vec<u8, A>) {
+    pub async fn attach_buffer(&self, buf: Vec<u8, D::IoBufferAllocator>) {
         let mut inner = self.inner.write().await;
 
         assert_eq!(inner.writers, 0);
@@ -264,7 +262,7 @@ where
         inner.attach_buffer(buf);
     }
 
-    pub async fn detach_buffer(&self) -> Vec<u8, A> {
+    pub async fn detach_buffer(&self) -> Vec<u8, D::IoBufferAllocator> {
         let mut inner = self.inner.write().await;
 
         inner.detach_buffer()
@@ -281,7 +279,7 @@ where
         can_write: bool,
         can_buffered_read: bool,
         can_physical_read: bool,
-    ) -> OwnedRwLockWriteGuard<RegionInner<A>> {
+    ) -> OwnedRwLockWriteGuard<RegionInner<D::IoBufferAllocator>> {
         self.inner
             .exclusive(can_write, can_buffered_read, can_physical_read)
             .await
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index e52a892d..f9ef5a21 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -12,17 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{marker::PhantomData, sync::Arc};
+use std::sync::Arc;
 
 use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{
-    core::adapter::Link, eviction::EvictionPolicy, intrusive_adapter, key_adapter, priority_adapter,
+    core::adapter::Link,
+    eviction::{EvictionPolicy, EvictionPolicyExt},
+    intrusive_adapter, key_adapter, priority_adapter,
 };
 use parking_lot::RwLock;
 use tokio::sync::RwLock as AsyncRwLock;
 
 use crate::{
-    device::{BufferAllocator, Device},
+    device::Device,
     flusher::{FlushTask, Flusher},
     reclaimer::{ReclaimTask, Reclaimer},
     region::{AllocateResult, Region, RegionId},
@@ -46,39 +48,36 @@ struct RegionManagerInner {
     current: Option<RegionId>,
 }
 
-pub struct RegionManager<A, D, E, EL>
+pub struct RegionManager<D, E, EL>
 where
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
-    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     inner: Arc<AsyncRwLock<RegionManagerInner>>,
 
-    buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+    buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
     clean_regions: Arc<AsyncQueue<RegionId>>,
 
-    regions: Vec<Region<A, D>>,
+    regions: Vec<Region<D>>,
     items: Vec<Arc<RegionEpItem<EL>>>,
 
     flusher: Arc<Flusher>,
     reclaimer: Arc<Reclaimer>,
 
     eviction: RwLock<E>,
-    _marker: PhantomData<EL>,
 }
 
-impl<A, D, E, EL> RegionManager<A, D, E, EL>
+impl<D, E, EL> RegionManager<D, E, EL>
 where
-    A: BufferAllocator,
-    D: Device<IoBufferAllocator = A>,
-    E: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     pub fn new(
         region_nums: usize,
         eviction_config: E::Config,
-        buffers: Arc<AsyncQueue<Vec<u8, A>>>,
+        buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
         clean_regions: Arc<AsyncQueue<RegionId>>,
         device: D,
         flusher: Arc<Flusher>,
@@ -92,7 +91,7 @@ where
         for id in 0..region_nums as RegionId {
             let region = Region::new(id, device.clone());
             let item = Arc::new(RegionEpItem {
-                link: E::Link::default(),
+                link: EL::default(),
                 id,
                 priority: 0,
             });
@@ -112,7 +111,6 @@ where
             flusher,
             reclaimer,
             eviction: RwLock::new(eviction),
-            _marker: PhantomData,
         }
     }
 
@@ -165,7 +163,7 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn region(&self, id: &RegionId) -> &Region<A, D> {
+    pub fn region(&self, id: &RegionId) -> &Region<D> {
         &self.regions[*id as usize]
     }
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 24d0198a..91201d11 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -22,7 +22,7 @@ use std::{
 
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, queue::AsyncQueue, rate::RateLimiter};
-use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use foyer_intrusive::eviction::EvictionPolicy;
 use futures::Future;
 use itertools::Itertools;
 use parking_lot::Mutex;
@@ -31,7 +31,7 @@ use twox_hash::XxHash64;
 
 use crate::{
     admission::{AdmissionPolicy, Judges},
-    device::{BufferAllocator, Device},
+    device::Device,
     error::{Error, Result},
     event::EventListener,
     flusher::Flusher,
@@ -43,6 +43,7 @@ use crate::{
     reinsertion::ReinsertionPolicy,
 };
 use foyer_common::code::{Key, Value};
+use foyer_intrusive::core::adapter::Link;
 use std::hash::Hasher;
 
 const REGION_MAGIC: u64 = 0x19970327;
@@ -55,13 +56,12 @@ pub struct PrometheusConfig {
     pub namespace: Option<String>,
 }
 
-pub struct StoreConfig<K, V, D, EP, EL>
+pub struct StoreConfig<K, V, D, EP>
 where
     K: Key,
     V: Value,
     D: Device,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    EL: Link,
+    EP: EvictionPolicy,
 {
     pub eviction_config: EP::Config,
     pub device_config: D::Config,
@@ -77,13 +77,12 @@ where
     pub prometheus_config: PrometheusConfig,
 }
 
-impl<K, V, D, EP, EL> Debug for StoreConfig<K, V, D, EP, EL>
+impl<K, V, D, EP> Debug for StoreConfig<K, V, D, EP>
 where
     K: Key,
     V: Value,
     D: Device,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
-    EL: Link,
+    EP: EvictionPolicy,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("StoreConfig")
@@ -98,18 +97,17 @@ where
     }
 }
 
-pub struct Store<K, V, BA, D, EP, EL>
+pub struct Store<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     indices: Arc<Indices<K>>,
 
-    region_manager: Arc<RegionManager<BA, D, EP, EL>>,
+    region_manager: Arc<RegionManager<D, EP, EL>>,
 
     device: D,
 
@@ -126,16 +124,15 @@ where
     _marker: PhantomData<V>,
 }
 
-impl<K, V, BA, D, EP, EL> Store<K, V, BA, D, EP, EL>
+impl<K, V, D, EP, EL> Store<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    pub async fn open(config: StoreConfig<K, V, D, EP, EL>) -> Result<Arc<Self>> {
+    pub async fn open(config: StoreConfig<K, V, D, EP>) -> Result<Arc<Self>> {
         tracing::info!("open store with config:\n{:#?}", config);
 
         let device = D::open(config.device_config).await?;
@@ -334,7 +331,7 @@ where
 
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[tracing::instrument(skip(self))]
-    pub fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, BA, D, EP, EL> {
+    pub fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, D, EP, EL> {
         StoreWriter::new(self, key, weight)
     }
 
@@ -485,12 +482,12 @@ where
     /// return `true` if region is valid, otherwise `false`
     async fn recover_region(
         region_id: RegionId,
-        region_manager: Arc<RegionManager<BA, D, EP, EL>>,
+        region_manager: Arc<RegionManager<D, EP, EL>>,
         indices: Arc<Indices<K>>,
         event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
     ) -> Result<bool> {
         let region = region_manager.region(&region_id).clone();
-        let res = if let Some(mut iter) = RegionEntryIter::<K, V, BA, D>::open(region).await? {
+        let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
             while let Some(index) = iter.next().await? {
                 for listener in event_listeners.iter() {
                     listener.on_recover(&index.key).await?;
@@ -506,7 +503,7 @@ where
         Ok(res)
     }
 
-    fn judge_inner(&self, writer: &StoreWriter<'_, K, V, BA, D, EP, EL>) -> Judges {
+    fn judge_inner(&self, writer: &StoreWriter<'_, K, V, D, EP, EL>) -> Judges {
         let mut res = Judges::new();
         for (index, admission) in self.admissions.iter().enumerate() {
             let admitted = admission.judge(&writer.key, writer.weight, &self.metrics);
@@ -517,7 +514,7 @@ where
 
     async fn apply_writer(
         &self,
-        mut writer: StoreWriter<'_, K, V, BA, D, EP, EL>,
+        mut writer: StoreWriter<'_, K, V, D, EP, EL>,
         value: V,
     ) -> Result<bool> {
         debug_assert!(!writer.inserted);
@@ -586,16 +583,15 @@ where
     }
 }
 
-pub struct StoreWriter<'a, K, V, BA, D, EP, EL>
+pub struct StoreWriter<'a, K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    store: &'a Store<K, V, BA, D, EP, EL>,
+    store: &'a Store<K, V, D, EP, EL>,
     key: K,
     weight: usize,
 
@@ -612,16 +608,15 @@ where
     inserted: bool,
 }
 
-impl<'a, K, V, BA, D, EP, EL> StoreWriter<'a, K, V, BA, D, EP, EL>
+impl<'a, K, V, D, EP, EL> StoreWriter<'a, K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    fn new(store: &'a Store<K, V, BA, D, EP, EL>, key: K, weight: usize) -> Self {
+    fn new(store: &'a Store<K, V, D, EP, EL>, key: K, weight: usize) -> Self {
         Self {
             store,
             key,
@@ -676,13 +671,12 @@ where
     }
 }
 
-impl<'a, K, V, BA, D, EP, EL> Debug for StoreWriter<'a, K, V, BA, D, EP, EL>
+impl<'a, K, V, D, EP, EL> Debug for StoreWriter<'a, K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -696,13 +690,12 @@ where
     }
 }
 
-impl<'a, K, V, BA, D, EP, EL> Drop for StoreWriter<'a, K, V, BA, D, EP, EL>
+impl<'a, K, V, D, EP, EL> Drop for StoreWriter<'a, K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
-    EP: EvictionPolicy<RegionEpItemAdapter<EL>, Link = EL>,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     fn drop(&mut self) {
@@ -878,28 +871,26 @@ fn checksum(buf: &[u8]) -> u64 {
     hasher.finish()
 }
 
-struct RegionEntryIter<K, V, BA, D>
+struct RegionEntryIter<K, V, D>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
+    D: Device,
 {
-    region: Region<BA, D>,
+    region: Region<D>,
 
     cursor: usize,
 
     _marker: PhantomData<(K, V)>,
 }
 
-impl<K, V, BA, D> RegionEntryIter<K, V, BA, D>
+impl<K, V, D> RegionEntryIter<K, V, D>
 where
     K: Key,
     V: Value,
-    BA: BufferAllocator,
-    D: Device<IoBufferAllocator = BA>,
+    D: Device,
 {
-    async fn open(region: Region<BA, D>) -> Result<Option<Self>> {
+    async fn open(region: Region<D>) -> Result<Option<Self>> {
         let region_size = region.device().region_size();
         let align = region.device().align();
 
@@ -984,24 +975,13 @@ pub mod tests {
 
     use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
 
-    use crate::device::{
-        allocator::AlignedAllocator,
-        fs::{FsDevice, FsDeviceConfig},
-    };
+    use crate::device::fs::{FsDevice, FsDeviceConfig};
 
     use super::*;
 
-    type TestStore = Store<
-        u64,
-        Vec<u8>,
-        AlignedAllocator,
-        FsDevice,
-        Fifo<RegionEpItemAdapter<FifoLink>>,
-        FifoLink,
-    >;
+    type TestStore = Store<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
-    type TestStoreConfig =
-        StoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
+    type TestStoreConfig = StoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
     #[tokio::test]
     #[allow(clippy::identity_op)]
@@ -1102,7 +1082,6 @@ pub mod tests {
             'static,
             u64,
             Vec<u8>,
-            AlignedAllocator,
             FsDevice,
             Fifo<RegionEpItemAdapter<FifoLink>>,
             FifoLink,

From 68d761d3b5ee25dcba5f9581a89dcbb71c036f79 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 26 Jul 2023 12:16:12 +0800
Subject: [PATCH 065/261] chore: add README and pr template (#88)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/pull_request_template.md | 23 +++++++++++++++++++++++
 README.md                        | 23 +++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 .github/pull_request_template.md
 create mode 100644 README.md

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 00000000..e33609bd
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,23 @@
+## What's changed and what's your intention?
+
+<!--
+
+**Please do not leave this empty!**
+
+Please explain **IN DETAIL** what the changes are in this PR and why they are needed:
+
+- Summarize your change (**mandatory**)
+- How does this PR work? Need a brief introduction for the changed logic (optional)
+- Describe clearly one logical change and avoid lazy messages (optional)
+- Describe any limitations of the current code (optional)
+- Refer to a related PR or issue link (optional)
+
+-->
+
+## Checklist
+
+- [ ] I have written necessary rustdoc comments
+- [ ] I have added necessary unit tests and integration tests
+- [ ] I have passed `make check` and `make test` in my local envirorment.
+
+## Related issues or PRs (optional)
diff --git a/README.md b/README.md
new file mode 100644
index 00000000..93e2dd59
--- /dev/null
+++ b/README.md
@@ -0,0 +1,23 @@
+# foyer
+
+*foyer* aims to be a user-friendly hybrid cache lib in Rust. 
+
+*foyer* is inspired by [Facebook/CacheLib](https://github.com/facebook/cachelib), which is an excellent hybrid cache lib in C++. *foyer* is not only a 'rewrite in Rust project', but provide some features that *CacheLib* doesn't have for now.
+
+## Development state
+
+Currently, *foyer* only finished few features, and is still under heavy development.
+
+## Roadmap
+
+- [ ] Better intrusive index collectios for memory cache.
+- [ ] Hybrid memory and disk cache.
+- [ ] Reinsertion for disk cache.
+- [ ] Raw device and single file device support.
+- [ ] More detailed metrics or statistics.
+
+## Contributing
+
+Contributions for *foyer* are welcomed! Issues can be found [here](https://github.com/MrCroxx/foyer/issues).
+
+Make sure you've passed `make check` and `make test` before request a review, or CI will fail.

From d88466c3ffc5e677a12dcd07719b0e30566f3b40 Mon Sep 17 00:00:00 2001
From: William Wen <44139337+wenym1@users.noreply.github.com>
Date: Wed, 26 Jul 2023 15:13:13 +0800
Subject: [PATCH 066/261] feat: implement async queue with notify (#86)

* feat: implement async queue with notify

* fix comment and add unit test

* fix fmt
---
 foyer-common/Cargo.toml   |  9 +++-
 foyer-common/src/queue.rs | 88 ++++++++++++++++++++++++++++-----------
 2 files changed, 71 insertions(+), 26 deletions(-)

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 45d70143..9f249b3c 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -11,7 +11,14 @@ license = "Apache-2.0"
 bytes = "1"
 parking_lot = "0.12"
 paste = "1.0"
-tokio = { version = "1", features = ["sync"] }
+tokio = { version = "1", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
 
 [dev-dependencies]
 rand = "0.8.5"
diff --git a/foyer-common/src/queue.rs b/foyer-common/src/queue.rs
index 832d6856..3727c47a 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/queue.rs
@@ -12,22 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    sync::atomic::{AtomicUsize, Ordering},
-};
-
-use tokio::sync::{
-    mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
-    Mutex,
-};
+use parking_lot::RwLock;
+use std::{collections::VecDeque, fmt::Debug};
+use tokio::sync::Notify;
 
 #[derive(Debug)]
-pub struct AsyncQueue<T: Debug> {
-    tx: UnboundedSender<T>,
-    rx: Mutex<UnboundedReceiver<T>>,
-
-    size: AtomicUsize,
+pub struct AsyncQueue<T> {
+    queue: RwLock<VecDeque<T>>,
+    notified: Notify,
 }
 
 impl<T: Debug> Default for AsyncQueue<T> {
@@ -38,31 +30,77 @@ impl<T: Debug> Default for AsyncQueue<T> {
 
 impl<T: Debug> AsyncQueue<T> {
     pub fn new() -> Self {
-        let (tx, rx) = unbounded_channel();
         Self {
-            tx,
-            rx: Mutex::new(rx),
-            size: AtomicUsize::new(0),
+            queue: RwLock::new(VecDeque::default()),
+            notified: Notify::new(),
         }
     }
 
     pub async fn acquire(&self) -> T {
-        let mut rx = self.rx.lock().await;
-        let item = rx.recv().await.unwrap();
-        self.size.fetch_sub(1, Ordering::Relaxed);
-        item
+        loop {
+            let notified = self.notified.notified();
+            {
+                let mut guard = self.queue.write();
+                if let Some(item) = guard.pop_front() {
+                    if !guard.is_empty() {
+                        // Since in `release` we use `notify_one`, not all waiters
+                        // will be waken up. Therefore if we figure out that the queue is not empty,
+                        // we call `notify_one` to awake the next pending `acquire`.
+                        self.notified.notify_one();
+                    }
+                    break item;
+                }
+            }
+            notified.await;
+        }
     }
 
     pub fn release(&self, item: T) {
-        self.size.fetch_add(1, Ordering::Relaxed);
-        self.tx.send(item).unwrap();
+        self.queue.write().push_back(item);
+        self.notified.notify_one();
     }
 
     pub fn len(&self) -> usize {
-        self.size.load(Ordering::Relaxed)
+        self.queue.read().len()
     }
 
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::queue::AsyncQueue;
+    use std::{
+        future::{poll_fn, Future},
+        pin::pin,
+        task::{Poll, Poll::Pending},
+    };
+
+    #[tokio::test]
+    async fn test_basic() {
+        let queue = AsyncQueue::new();
+        queue.release(1);
+        assert_eq!(1, queue.acquire().await);
+    }
+
+    #[tokio::test]
+    async fn test_multiple_reader() {
+        let queue = AsyncQueue::new();
+        let mut read_future1 = pin!(queue.acquire());
+        let mut read_future2 = pin!(queue.acquire());
+        assert_eq!(
+            Pending,
+            poll_fn(|cx| Poll::Ready(read_future1.as_mut().poll(cx))).await
+        );
+        assert_eq!(
+            Pending,
+            poll_fn(|cx| Poll::Ready(read_future2.as_mut().poll(cx))).await
+        );
+        queue.release(1);
+        queue.release(2);
+        assert_eq!(1, read_future1.await);
+        assert_eq!(2, read_future2.await);
+    }
+}

From 02b7761baee85fc9c8426fc3c238aef1eb1d1140 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 27 Jul 2023 20:36:14 +0800
Subject: [PATCH 067/261] feat: impl reinsertion (#89)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml                       |   4 +-
 foyer-common/src/lib.rs                       |   1 +
 foyer-common/src/queue.rs                     |  15 +
 foyer-common/src/rated_random.rs              | 245 ++++++++++++++
 foyer-storage-bench/src/main.rs               |  27 +-
 foyer-storage/src/admission/mod.rs            |   7 +-
 foyer-storage/src/admission/rated_random.rs   | 245 +-------------
 foyer-storage/src/judge.rs                    |  92 ++++++
 foyer-storage/src/lib.rs                      |   1 +
 foyer-storage/src/reclaimer.rs                |  97 ++++--
 foyer-storage/src/reinsertion/exist.rs        |  89 +++++
 .../{reinsertion.rs => reinsertion/mod.rs}    |  15 +-
 foyer-storage/src/reinsertion/rated_random.rs |  67 ++++
 foyer-storage/src/store.rs                    | 304 +++++++++++-------
 14 files changed, 839 insertions(+), 370 deletions(-)
 create mode 100644 foyer-common/src/rated_random.rs
 create mode 100644 foyer-storage/src/judge.rs
 create mode 100644 foyer-storage/src/reinsertion/exist.rs
 rename foyer-storage/src/{reinsertion.rs => reinsertion/mod.rs} (60%)
 create mode 100644 foyer-storage/src/reinsertion/rated_random.rs

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 9f249b3c..1019541d 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -11,6 +11,7 @@ license = "Apache-2.0"
 bytes = "1"
 parking_lot = "0.12"
 paste = "1.0"
+rand = "0.8.5"
 tokio = { version = "1", features = [
     "rt",
     "rt-multi-thread",
@@ -19,6 +20,7 @@ tokio = { version = "1", features = [
     "time",
     "signal",
 ] }
+tracing = "0.1"
 
 [dev-dependencies]
-rand = "0.8.5"
+itertools = "0.10.5"
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 1a64d5ef..02f0e9d6 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -18,3 +18,4 @@ pub mod bits;
 pub mod code;
 pub mod queue;
 pub mod rate;
+pub mod rated_random;
diff --git a/foyer-common/src/queue.rs b/foyer-common/src/queue.rs
index 3727c47a..763ca066 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/queue.rs
@@ -36,6 +36,21 @@ impl<T: Debug> AsyncQueue<T> {
         }
     }
 
+    pub fn try_acquire(&self) -> Option<T> {
+        let mut guard = self.queue.write();
+        if let Some(item) = guard.pop_front() {
+            if !guard.is_empty() {
+                // Since in `release` we use `notify_one`, not all waiters
+                // will be waken up. Therefore if we figure out that the queue is not empty,
+                // we call `notify_one` to awake the next pending `acquire`.
+                self.notified.notify_one();
+            }
+            Some(item)
+        } else {
+            None
+        }
+    }
+
     pub async fn acquire(&self) -> T {
         loop {
             let notified = self.notified.notified();
diff --git a/foyer-common/src/rated_random.rs b/foyer-common/src/rated_random.rs
new file mode 100644
index 00000000..dfd76811
--- /dev/null
+++ b/foyer-common/src/rated_random.rs
@@ -0,0 +1,245 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    sync::atomic::{AtomicUsize, Ordering},
+    time::{Duration, Instant},
+};
+
+use crate::code::{Key, Value};
+use parking_lot::{Mutex, MutexGuard};
+use rand::{thread_rng, Rng};
+
+const PRECISION: usize = 100000;
+
+/// p       : admit probability among △t
+/// w_t    :  entry weight at time t
+/// r       : actual admitted rate
+/// E(w)    : expected admitted weight
+/// E(r)    : expceted admitted rate
+///
+/// E(w_t) = {
+///   p * w  ( if  judge &&  insert )
+///   p * w  ( if !judge && !insert )
+///   w      ( if !judge &&  insert )
+///   0      ( if  judge && !insert )
+/// }
+///
+///    E(r) = sum_{△t}{E(w_t)} / △t
+/// => E(r) * △t = sum_{△t}{ E(w_t) }
+/// => E(r) * △t =   sum_{△t}^{(judge && insert) || (!judge && !insert)}{ p * w_t }
+///                + sum_{△t}^{(!judge && insert)}{ w_t }
+///                + sum_{△t}^{(judge && !insert){ 0 }
+/// => E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t } = p * sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t }
+/// => p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
+///
+/// p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
+///     ↑ rate        ↑ △force_insert_bytes                   ↑ △obey_bytes
+///
+/// p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
+#[derive(Debug)]
+pub struct RatedRandom<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    rate: usize,
+    update_interval: Duration,
+
+    obey_bytes: AtomicUsize,
+    force_inser_bytes: AtomicUsize,
+
+    probability: AtomicUsize,
+
+    inner: Mutex<Inner>,
+
+    _marker: PhantomData<(K, V)>,
+}
+
+#[derive(Debug)]
+struct Inner {
+    last_update_time: Option<Instant>,
+    last_obey_bytes: usize,
+    last_force_insert_bytes: usize,
+}
+
+impl<K, V> RatedRandom<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn new(rate: usize, update_interval: Duration) -> Self {
+        Self {
+            rate,
+            update_interval,
+
+            obey_bytes: AtomicUsize::new(0),
+            force_inser_bytes: AtomicUsize::new(0),
+
+            probability: AtomicUsize::new(0),
+            inner: Mutex::new(Inner {
+                last_update_time: None,
+                last_obey_bytes: 0,
+                last_force_insert_bytes: 0,
+            }),
+            _marker: PhantomData,
+        }
+    }
+
+    pub fn judge(&self, _key: &K, _weight: usize) -> bool {
+        if let Some(inner) = self.inner.try_lock() {
+            self.update(inner);
+        }
+
+        thread_rng().gen_range(0..PRECISION) < self.probability.load(Ordering::Relaxed)
+    }
+
+    pub fn on_insert(&self, _key: &K, weight: usize, judge: bool) {
+        if judge {
+            // obey
+            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
+        } else {
+            // force insert
+            self.force_inser_bytes.fetch_add(weight, Ordering::Relaxed);
+        }
+    }
+
+    pub fn on_drop(&self, _key: &K, weight: usize, judge: bool) {
+        if !judge {
+            // obey
+            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
+        }
+    }
+
+    fn update(&self, mut inner: MutexGuard<'_, Inner>) {
+        // p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
+
+        let now = Instant::now();
+
+        let elapsed = match inner.last_update_time {
+            Some(last_update_time) => now.duration_since(last_update_time),
+            None => self.update_interval,
+        };
+
+        if elapsed < self.update_interval {
+            return;
+        }
+
+        let now_obey_bytes = self.obey_bytes.load(Ordering::Relaxed);
+        let now_force_insert_bytes = self.force_inser_bytes.load(Ordering::Relaxed);
+
+        let delta_obey_bytes = now_obey_bytes - inner.last_obey_bytes;
+        let delta_force_insert_bytes = now_force_insert_bytes - inner.last_force_insert_bytes;
+
+        inner.last_update_time = Some(now);
+        inner.last_obey_bytes = now_obey_bytes;
+        inner.last_force_insert_bytes = now_force_insert_bytes;
+
+        let p = if delta_obey_bytes == 0 {
+            1.0
+        } else {
+            let numerator =
+                self.rate as f64 * elapsed.as_secs_f64() - delta_force_insert_bytes as f64;
+            let numerator = numerator.abs();
+            let p = numerator / delta_obey_bytes as f64;
+            p.min(1.0)
+        };
+
+        debug_assert!((0.0..=1.0).contains(&p), "p out of range 0..=1: {}", p);
+
+        let p = (p * PRECISION as f64) as usize;
+        self.probability.store(p, Ordering::Relaxed);
+
+        tracing::debug!("probability: {}", p as f64 / PRECISION as f64);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use itertools::Itertools;
+
+    use super::*;
+
+    #[ignore]
+    #[tokio::test]
+    async fn test_rated_random() {
+        const CASES: usize = 10;
+        const ERATIO: f64 = 0.1;
+
+        let handles = (0..CASES).map(|_| tokio::spawn(case())).collect_vec();
+        let mut eratios = vec![];
+        for handle in handles {
+            let eratio = handle.await.unwrap();
+            assert!(eratio < ERATIO, "eratio: {} < ERATIO: {}", eratio, ERATIO);
+            eratios.push(eratio);
+        }
+        println!("========== RatedRandom error ratio begin ==========");
+        for eratio in eratios {
+            println!("eratio: {eratio}");
+        }
+        println!("=========== RatedRandom error ratio end ===========");
+    }
+
+    async fn case() -> f64 {
+        const RATE: usize = 1_000_000;
+        const CONCURRENCY: usize = 10;
+
+        const P_OTHER: f64 = 0.8;
+        const P_FORCE: f64 = 0.1;
+
+        let score = Arc::new(AtomicUsize::new(0));
+
+        let rr = Arc::new(RatedRandom::<u64, Vec<u8>>::new(
+            RATE,
+            Duration::from_millis(100),
+        ));
+
+        // scope: CONCURRENCY * (1 / interval) * range
+        // [1_000_000, 10_000_000]
+        // FORCE: [100_000, 1_000_000]
+
+        async fn submit(rr: Arc<RatedRandom<u64, Vec<u8>>>, score: Arc<AtomicUsize>) {
+            loop {
+                tokio::time::sleep(Duration::from_millis(1)).await;
+                let weight = thread_rng().gen_range(100..1000);
+
+                let judge = rr.judge(&0, weight);
+                let p_other = thread_rng().gen_range(0.0..=1.0);
+                let p_force = thread_rng().gen_range(0.0..=1.0);
+
+                let insert = (p_force <= P_FORCE) || (p_other <= P_OTHER && judge);
+
+                if insert {
+                    score.fetch_add(weight, Ordering::Relaxed);
+                    rr.on_insert(&0, weight, judge);
+                } else {
+                    rr.on_drop(&0, weight, judge);
+                }
+            }
+        }
+
+        for _ in 0..CONCURRENCY {
+            tokio::spawn(submit(rr.clone(), score.clone()));
+        }
+
+        tokio::time::sleep(Duration::from_secs(10)).await;
+        let s = score.load(Ordering::Relaxed);
+        let error = (s as isize - RATE as isize * 10).unsigned_abs();
+        error as f64 / (RATE as f64 * 10.0)
+    }
+}
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index c1215af2..67372586 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -33,8 +33,9 @@ use clap::Parser;
 
 use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
-    admission::{rated_random::RatedRandom, AdmissionPolicy},
+    admission::{rated_random::RatedRandomAdmissionPolicy, AdmissionPolicy},
     device::fs::FsDeviceConfig,
+    reinsertion::{rated_random::RatedRandomReinsertionPolicy, ReinsertionPolicy},
     store::{PrometheusConfig, StoreConfig},
     LfuFsStore,
 };
@@ -119,7 +120,12 @@ pub struct Args {
     /// enable rated random admission policy if `rated_random` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
-    rated_random: usize,
+    rated_random_admission: usize,
+
+    /// enable rated random admission policy if `rated_random` > 0
+    /// (MiB/s)
+    #[arg(long, default_value_t = 0)]
+    rated_random_reinsertion: usize,
 
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
@@ -221,16 +227,27 @@ async fn main() {
     };
 
     let mut admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
-    if args.rated_random > 0 {
-        let rr = RatedRandom::new(args.rated_random * 1024 * 1024, Duration::from_millis(100));
+    let mut reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
+    if args.rated_random_admission > 0 {
+        let rr = RatedRandomAdmissionPolicy::new(
+            args.rated_random_admission * 1024 * 1024,
+            Duration::from_millis(100),
+        );
         admissions.push(Arc::new(rr));
     }
+    if args.rated_random_reinsertion > 0 {
+        let rr = RatedRandomReinsertionPolicy::new(
+            args.rated_random_reinsertion * 1024 * 1024,
+            Duration::from_millis(100),
+        );
+        reinsertions.push(Arc::new(rr));
+    }
 
     let config = StoreConfig {
         eviction_config,
         device_config,
         admissions,
-        reinsertions: vec![],
+        reinsertions,
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
         flushers: args.flushers,
         flush_rate_limit: args.flush_rate_limit * 1024 * 1024,
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 57dab869..54fb2092 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -12,20 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use bitmaps::Bitmap;
 use foyer_common::code::{Key, Value};
 
 use std::{fmt::Debug, sync::Arc};
 
-use crate::metrics::Metrics;
-
-pub type Judges = Bitmap<64>;
+use crate::{indices::Indices, metrics::Metrics};
 
 #[allow(unused_variables)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
+    fn init(&self, indices: &Arc<Indices<Self::Key>>) {}
+
     fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
 
     fn on_insert(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
index adba2e69..79675754 100644
--- a/foyer-storage/src/admission/rated_random.rs
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -12,169 +12,39 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    marker::PhantomData,
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc,
-    },
-    time::{Duration, Instant},
-};
+use std::{fmt::Debug, sync::Arc, time::Duration};
 
-use foyer_common::code::{Key, Value};
-use parking_lot::{Mutex, MutexGuard};
-use rand::{thread_rng, Rng};
+use foyer_common::{
+    code::{Key, Value},
+    rated_random::RatedRandom,
+};
 
 use crate::metrics::Metrics;
 
 use super::AdmissionPolicy;
 
-const PRECISION: usize = 100000;
-
-/// p       : admit probability among △t
-/// w_t    :  entry weight at time t
-/// r       : actual admitted rate
-/// E(w)    : expected admitted weight
-/// E(r)    : expceted admitted rate
-///
-/// E(w_t) = {
-///   p * w  ( if  judge &&  insert )
-///   p * w  ( if !judge && !insert )
-///   w      ( if !judge &&  insert )
-///   0      ( if  judge && !insert )
-/// }
-///
-///    E(r) = sum_{△t}{E(w_t)} / △t
-/// => E(r) * △t = sum_{△t}{ E(w_t) }
-/// => E(r) * △t =   sum_{△t}^{(judge && insert) || (!judge && !insert)}{ p * w_t }
-///                + sum_{△t}^{(!judge && insert)}{ w_t }
-///                + sum_{△t}^{(judge && !insert){ 0 }
-/// => E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t } = p * sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t }
-/// => p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
-///
-/// p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
-///     ↑ rate        ↑ △force_insert_bytes                   ↑ △obey_bytes
-///
-/// p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
 #[derive(Debug)]
-pub struct RatedRandom<K, V>
+pub struct RatedRandomAdmissionPolicy<K, V>
 where
     K: Key,
     V: Value,
 {
-    rate: usize,
-    update_interval: Duration,
-
-    obey_bytes: AtomicUsize,
-    force_inser_bytes: AtomicUsize,
-
-    probability: AtomicUsize,
-
-    inner: Mutex<Inner>,
-
-    _marker: PhantomData<(K, V)>,
+    inner: RatedRandom<K, V>,
 }
 
-#[derive(Debug)]
-struct Inner {
-    last_update_time: Option<Instant>,
-    last_obey_bytes: usize,
-    last_force_insert_bytes: usize,
-}
-
-impl<K, V> RatedRandom<K, V>
+impl<K, V> RatedRandomAdmissionPolicy<K, V>
 where
     K: Key,
     V: Value,
 {
     pub fn new(rate: usize, update_interval: Duration) -> Self {
         Self {
-            rate,
-            update_interval,
-
-            obey_bytes: AtomicUsize::new(0),
-            force_inser_bytes: AtomicUsize::new(0),
-
-            probability: AtomicUsize::new(0),
-            inner: Mutex::new(Inner {
-                last_update_time: None,
-                last_obey_bytes: 0,
-                last_force_insert_bytes: 0,
-            }),
-            _marker: PhantomData,
-        }
-    }
-
-    fn judge(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
-        if let Some(inner) = self.inner.try_lock() {
-            self.update(inner);
-        }
-
-        thread_rng().gen_range(0..PRECISION) < self.probability.load(Ordering::Relaxed)
-    }
-
-    fn on_insert(&self, _key: &K, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
-        if judge {
-            // obey
-            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
-        } else {
-            // force insert
-            self.force_inser_bytes.fetch_add(weight, Ordering::Relaxed);
+            inner: RatedRandom::new(rate, update_interval),
         }
     }
-
-    fn on_drop(&self, _key: &K, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
-        if !judge {
-            // obey
-            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
-        }
-    }
-
-    fn update(&self, mut inner: MutexGuard<'_, Inner>) {
-        // p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
-
-        let now = Instant::now();
-
-        let elapsed = match inner.last_update_time {
-            Some(last_update_time) => now.duration_since(last_update_time),
-            None => self.update_interval,
-        };
-
-        if elapsed < self.update_interval {
-            return;
-        }
-
-        let now_obey_bytes = self.obey_bytes.load(Ordering::Relaxed);
-        let now_force_insert_bytes = self.force_inser_bytes.load(Ordering::Relaxed);
-
-        let delta_obey_bytes = now_obey_bytes - inner.last_obey_bytes;
-        let delta_force_insert_bytes = now_force_insert_bytes - inner.last_force_insert_bytes;
-
-        inner.last_update_time = Some(now);
-        inner.last_obey_bytes = now_obey_bytes;
-        inner.last_force_insert_bytes = now_force_insert_bytes;
-
-        let p = if delta_obey_bytes == 0 {
-            1.0
-        } else {
-            let numerator =
-                self.rate as f64 * elapsed.as_secs_f64() - delta_force_insert_bytes as f64;
-            let numerator = numerator.abs();
-            let p = numerator / delta_obey_bytes as f64;
-            p.min(1.0)
-        };
-
-        debug_assert!((0.0..=1.0).contains(&p), "p out of range 0..=1: {}", p);
-
-        let p = (p * PRECISION as f64) as usize;
-        self.probability.store(p, Ordering::Relaxed);
-
-        tracing::debug!("probability: {}", p as f64 / PRECISION as f64);
-    }
 }
 
-impl<K, V> AdmissionPolicy for RatedRandom<K, V>
+impl<K, V> AdmissionPolicy for RatedRandomAdmissionPolicy<K, V>
 where
     K: Key,
     V: Value,
@@ -183,98 +53,15 @@ where
 
     type Value = V;
 
-    fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool {
-        self.judge(key, weight, metrics)
-    }
-
-    fn on_insert(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool) {
-        self.on_insert(key, weight, metrics, judge)
+    fn judge(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.inner.judge(key, weight)
     }
 
-    fn on_drop(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool) {
-        self.on_drop(key, weight, metrics, judge)
+    fn on_insert(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_insert(key, weight, judge)
     }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use itertools::Itertools;
-
-    use super::*;
-
-    #[ignore]
-    #[tokio::test]
-    async fn test_rated_random() {
-        const CASES: usize = 10;
-        const ERATIO: f64 = 0.1;
-
-        let handles = (0..CASES).map(|_| tokio::spawn(case())).collect_vec();
-        let mut eratios = vec![];
-        for handle in handles {
-            let eratio = handle.await.unwrap();
-            assert!(eratio < ERATIO, "eratio: {} < ERATIO: {}", eratio, ERATIO);
-            eratios.push(eratio);
-        }
-        println!("========== RatedRandom error ratio begin ==========");
-        for eratio in eratios {
-            println!("eratio: {eratio}");
-        }
-        println!("=========== RatedRandom error ratio end ===========");
-    }
-
-    async fn case() -> f64 {
-        const RATE: usize = 1_000_000;
-        const CONCURRENCY: usize = 10;
-
-        const P_OTHER: f64 = 0.8;
-        const P_FORCE: f64 = 0.1;
-
-        let metrics = Arc::new(Metrics::default());
-
-        let score = Arc::new(AtomicUsize::new(0));
-
-        let rr = Arc::new(RatedRandom::<u64, Vec<u8>>::new(
-            RATE,
-            Duration::from_millis(100),
-        ));
-
-        // scope: CONCURRENCY * (1 / interval) * range
-        // [1_000_000, 10_000_000]
-        // FORCE: [100_000, 1_000_000]
-
-        async fn submit(
-            rr: Arc<RatedRandom<u64, Vec<u8>>>,
-            score: Arc<AtomicUsize>,
-            metrics: Arc<Metrics>,
-        ) {
-            loop {
-                tokio::time::sleep(Duration::from_millis(1)).await;
-                let weight = thread_rng().gen_range(100..1000);
-
-                let judge = rr.judge(&0, weight, &metrics);
-                let p_other = thread_rng().gen_range(0.0..=1.0);
-                let p_force = thread_rng().gen_range(0.0..=1.0);
-
-                let insert = (p_force <= P_FORCE) || (p_other <= P_OTHER && judge);
-
-                if insert {
-                    score.fetch_add(weight, Ordering::Relaxed);
-                    rr.on_insert(&0, weight, &metrics, judge);
-                } else {
-                    rr.on_drop(&0, weight, &metrics, judge);
-                }
-            }
-        }
-
-        for _ in 0..CONCURRENCY {
-            tokio::spawn(submit(rr.clone(), score.clone(), metrics.clone()));
-        }
 
-        tokio::time::sleep(Duration::from_secs(10)).await;
-        let s = score.load(Ordering::Relaxed);
-        let error = (s as isize - RATE as isize * 10).unsigned_abs();
-        error as f64 / (RATE as f64 * 10.0)
+    fn on_drop(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_drop(key, weight, judge)
     }
 }
diff --git a/foyer-storage/src/judge.rs b/foyer-storage/src/judge.rs
new file mode 100644
index 00000000..069498ff
--- /dev/null
+++ b/foyer-storage/src/judge.rs
@@ -0,0 +1,92 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::ops::{BitAnd, BitOr};
+
+use bitmaps::Bitmap;
+
+#[derive(Debug)]
+pub struct Judges {
+    /// 1: admit
+    /// 0: reject
+    judge: Bitmap<64>,
+    /// 1: use
+    /// 0: ignore
+    umask: Bitmap<64>,
+}
+
+impl Judges {
+    pub fn new(size: usize) -> Self {
+        let mut mask = Bitmap::default();
+        mask.invert();
+        Self::with_mask(size, mask)
+    }
+
+    pub fn with_mask(size: usize, mask: Bitmap<64>) -> Self {
+        let mut umask = mask.bitand(Bitmap::from_value(
+            1u64.wrapping_shl(size as u32).wrapping_sub(1),
+        ));
+        umask.invert();
+
+        Self {
+            judge: Bitmap::default(),
+            umask,
+        }
+    }
+
+    pub fn get(&mut self, index: usize) -> bool {
+        self.judge.get(index)
+    }
+
+    pub fn set(&mut self, index: usize, judge: bool) {
+        self.judge.set(index, judge);
+    }
+
+    pub fn apply(&mut self, judge: Bitmap<64>) {
+        self.judge = judge;
+    }
+
+    /// judge | ( ~mask )
+    ///
+    /// | judge | mask | ~mask | result |
+    /// |   0   |  0   |   1   |    1   |
+    /// |   0   |  1   |   0   |    0   |
+    /// |   1   |  0   |   1   |    1   |
+    /// |   1   |  1   |   0   |    1   |
+    pub fn judge(&self) -> bool {
+        self.judge.bitor(self.umask).is_full()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_judge() {
+        let mask = Bitmap::from_value(0b_0011);
+
+        let dataset = vec![
+            (mask, Bitmap::from_value(0b_0011), true),
+            (mask, Bitmap::from_value(0b_1011), true),
+            (mask, Bitmap::from_value(0b_1010), false),
+        ];
+
+        for (i, (mask, j, e)) in dataset.into_iter().enumerate() {
+            let mut judge = Judges::with_mask(4, mask);
+            judge.apply(j);
+            assert_eq!(judge.judge(), e, "case {}, {} != {}", i, judge.judge(), e);
+        }
+    }
+}
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index c2e5fb52..e1370c2e 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -25,6 +25,7 @@ pub mod error;
 pub mod event;
 pub mod flusher;
 pub mod indices;
+pub mod judge;
 pub mod metrics;
 pub mod reclaimer;
 pub mod region;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index dd633f6e..caa9148a 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -19,11 +19,12 @@ use crate::{
     error::{Error, Result},
     event::EventListener,
     indices::Indices,
+    judge::Judges,
     metrics::Metrics,
     region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
-    store::Store,
+    store::{RegionEntryIter, Store},
 };
 use bytes::BufMut;
 use foyer_common::{
@@ -102,12 +103,12 @@ impl Reclaimer {
             .zip_eq(stop_rxs.into_iter())
             .map(|(task_rx, stop_rx)| Runner {
                 task_rx,
-                _store: store.clone(),
+                store: store.clone(),
                 region_manager: region_manager.clone(),
                 clean_regions: clean_regions.clone(),
-                _reinsertions: reinsertions.clone(),
+                reinsertions: reinsertions.clone(),
                 indices: indices.clone(),
-                _rate_limiter: rate_limiter.clone(),
+                rate_limiter: rate_limiter.clone(),
                 stop_rx,
                 metrics: metrics.clone(),
                 event_listeners: event_listeners.clone(),
@@ -149,13 +150,13 @@ where
 {
     task_rx: mpsc::Receiver<ReclaimTask>,
 
-    _store: Arc<Store<K, V, D, EP, EL>>,
+    store: Arc<Store<K, V, D, EP, EL>>,
     region_manager: Arc<RegionManager<D, EP, EL>>,
     clean_regions: Arc<AsyncQueue<RegionId>>,
-    _reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
+    reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
     indices: Arc<Indices<K>>,
 
-    _rate_limiter: Option<Arc<RateLimiter>>,
+    rate_limiter: Option<Arc<RateLimiter>>,
 
     event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
 
@@ -204,18 +205,78 @@ where
         }
 
         // after drop indices and acquire exclusive lock, no writers or readers are supposed to access the region
-        let guard = region.exclusive(false, false, false).await;
-
-        tracing::trace!(
-            "[reclaimer] region {}, writers: {}, buffered readers: {}, physical readers: {}",
-            region.id(),
-            guard.writers(),
-            guard.buffered_readers(),
-            guard.physical_readers()
-        );
+        {
+            let guard = region.exclusive(false, false, false).await;
+            tracing::trace!(
+                "[reclaimer] region {}, writers: {}, buffered readers: {}, physical readers: {}",
+                region.id(),
+                guard.writers(),
+                guard.buffered_readers(),
+                guard.physical_readers()
+            );
+            drop(guard);
+        }
 
         // step 2: do reinsertion
-        // TODO(MrCroxx): do reinsertion
+        let reinsert = || {
+            let region = region.clone();
+            let metrics = self.metrics.clone();
+            let rate = self.rate_limiter.clone();
+            let reinsertions = self.reinsertions.clone();
+
+            tracing::info!("[reclaimer] begin reinsertion, region: {}", task.region_id);
+
+            async move {
+                let mut iter = match RegionEntryIter::<K, V, D>::open(region).await {
+                    Ok(Some(iter)) => iter,
+                    Ok(None) => return Ok(()),
+                    Err(e) => return Err(e),
+                };
+
+                while let Some((key, value)) = iter.next_kv().await? {
+                    let weight = key.serialized_len() + value.serialized_len();
+
+                    let mut judges = Judges::new(reinsertions.len());
+                    for (index, reinsertion) in reinsertions.iter().enumerate() {
+                        let judge = reinsertion.judge(&key, weight, &metrics);
+                        judges.set(index, judge);
+                    }
+                    if !judges.judge() {
+                        for (index, reinsertion) in reinsertions.iter().enumerate() {
+                            let judge = judges.get(index);
+                            reinsertion.on_drop(&key, weight, &metrics, judge);
+                        }
+                        continue;
+                    }
+
+                    if let Some(rate) = rate.as_ref() && let Some(wait) = rate.consume(weight as f64) {
+                        tokio::time::sleep(wait).await;
+                    }
+
+                    if self.store.insert(key.clone(), value).await? {
+                        for (index, reinsertion) in reinsertions.iter().enumerate() {
+                            let judge = judges.get(index);
+                            reinsertion.on_insert(&key, weight, &metrics, judge);
+                        }
+                    } else {
+                        for (index, reinsertion) in reinsertions.iter().enumerate() {
+                            let judge = judges.get(index);
+                            reinsertion.on_drop(&key, weight, &metrics, judge);
+                        }
+                    }
+
+                    metrics.bytes_reinsert.inc_by(weight as u64);
+                }
+
+                tracing::info!("[reclaimer] finish reinsertion, region: {}", task.region_id);
+
+                Ok(())
+            }
+        };
+
+        if  !self.reinsertions.is_empty() && let Err(e) = reinsert().await {
+            tracing::warn!("reinsert region {:?} error: {:?}", region, e);
+        }
 
         // step 3: set region last block zero
         let align = region.device().align();
@@ -230,8 +291,6 @@ where
         // step 4: send clean region
         self.clean_regions.release(task.region_id);
 
-        drop(guard);
-
         tracing::info!(
             "[reclaimer] finish reclaim task, region: {}",
             task.region_id
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
new file mode 100644
index 00000000..95500032
--- /dev/null
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -0,0 +1,89 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    marker::PhantomData,
+    sync::{Arc, OnceLock},
+};
+
+use foyer_common::code::{Key, Value};
+
+use crate::indices::Indices;
+
+use super::ReinsertionPolicy;
+
+#[derive(Debug)]
+pub struct ExistReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    indices: OnceLock<Arc<Indices<K>>>,
+    _marker: PhantomData<V>,
+}
+
+impl<K, V> Default for ExistReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn default() -> Self {
+        Self {
+            indices: OnceLock::new(),
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<K, V> ReinsertionPolicy for ExistReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+
+    type Value = V;
+
+    fn init(&self, indices: &Arc<crate::indices::Indices<Self::Key>>) {
+        self.indices.get_or_init(|| indices.clone());
+    }
+
+    fn judge(
+        &self,
+        key: &Self::Key,
+        _weight: usize,
+        _metrics: &std::sync::Arc<crate::metrics::Metrics>,
+    ) -> bool {
+        let indices = self.indices.get().unwrap();
+        indices.lookup(key).is_some()
+    }
+
+    fn on_insert(
+        &self,
+        _key: &Self::Key,
+        _weight: usize,
+        _metrics: &Arc<crate::metrics::Metrics>,
+        _judge: bool,
+    ) {
+    }
+
+    fn on_drop(
+        &self,
+        _key: &Self::Key,
+        _weight: usize,
+        _metrics: &Arc<crate::metrics::Metrics>,
+        _judge: bool,
+    ) {
+    }
+}
diff --git a/foyer-storage/src/reinsertion.rs b/foyer-storage/src/reinsertion/mod.rs
similarity index 60%
rename from foyer-storage/src/reinsertion.rs
rename to foyer-storage/src/reinsertion/mod.rs
index 679c935e..006301ce 100644
--- a/foyer-storage/src/reinsertion.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -14,11 +14,22 @@
 
 use foyer_common::code::{Key, Value};
 
-use std::fmt::Debug;
+use crate::{indices::Indices, metrics::Metrics};
+use std::{fmt::Debug, sync::Arc};
 
+#[allow(unused_variables)]
 pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn judge(&self, key: &Self::Key, value: &Self::Value) -> bool;
+    fn init(&self, indices: &Arc<Indices<Self::Key>>) {}
+
+    fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
+
+    fn on_insert(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
+
+    fn on_drop(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
 }
+
+pub mod exist;
+pub mod rated_random;
diff --git a/foyer-storage/src/reinsertion/rated_random.rs b/foyer-storage/src/reinsertion/rated_random.rs
new file mode 100644
index 00000000..7c570268
--- /dev/null
+++ b/foyer-storage/src/reinsertion/rated_random.rs
@@ -0,0 +1,67 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, sync::Arc, time::Duration};
+
+use foyer_common::{
+    code::{Key, Value},
+    rated_random::RatedRandom,
+};
+
+use crate::metrics::Metrics;
+
+use super::ReinsertionPolicy;
+
+#[derive(Debug)]
+pub struct RatedRandomReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    inner: RatedRandom<K, V>,
+}
+
+impl<K, V> RatedRandomReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn new(rate: usize, update_interval: Duration) -> Self {
+        Self {
+            inner: RatedRandom::new(rate, update_interval),
+        }
+    }
+}
+
+impl<K, V> ReinsertionPolicy for RatedRandomReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.inner.judge(key, weight)
+    }
+
+    fn on_insert(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_insert(key, weight, judge)
+    }
+
+    fn on_drop(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_drop(key, weight, judge)
+    }
+}
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 91201d11..056b4dd4 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -15,7 +15,6 @@
 use std::{
     fmt::Debug,
     marker::PhantomData,
-    ops::{BitAnd, BitOr},
     sync::Arc,
     time::{Duration, Instant},
 };
@@ -30,12 +29,13 @@ use tokio::{sync::broadcast, task::JoinHandle};
 use twox_hash::XxHash64;
 
 use crate::{
-    admission::{AdmissionPolicy, Judges},
+    admission::AdmissionPolicy,
     device::Device,
     error::{Error, Result},
     event::EventListener,
     flusher::Flusher,
     indices::{Index, Indices},
+    judge::Judges,
     metrics::Metrics,
     reclaimer::Reclaimer,
     region::{Region, RegionId},
@@ -194,6 +194,13 @@ where
             _marker: PhantomData,
         });
 
+        for admission in store.admissions.iter() {
+            admission.init(&store.indices);
+        }
+        for reinsertion in config.reinsertions.iter() {
+            reinsertion.init(&store.indices);
+        }
+
         let flush_rate_limiter = match config.flush_rate_limit {
             0 => None,
             rate => Some(Arc::new(RateLimiter::new(rate as f64))),
@@ -247,14 +254,14 @@ where
         Ok(())
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self, value))]
     pub async fn insert(&self, key: K, value: V) -> Result<bool> {
         let weight = key.serialized_len() + value.serialized_len();
         let writer = StoreWriter::new(self, key, weight);
         writer.finish(value).await
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self, value))]
     pub async fn insert_if_not_exists(&self, key: K, value: V) -> Result<bool> {
         if self.exists(&key)? {
             return Ok(false);
@@ -503,13 +510,12 @@ where
         Ok(res)
     }
 
-    fn judge_inner(&self, writer: &StoreWriter<'_, K, V, D, EP, EL>) -> Judges {
-        let mut res = Judges::new();
+    fn judge_inner(&self, writer: &mut StoreWriter<'_, K, V, D, EP, EL>) {
         for (index, admission) in self.admissions.iter().enumerate() {
-            let admitted = admission.judge(&writer.key, writer.weight, &self.metrics);
-            res.set(index, admitted);
+            let judge = admission.judge(&writer.key, writer.weight, &self.metrics);
+            writer.judges.set(index, judge);
         }
-        res
+        writer.is_judged = true;
     }
 
     async fn apply_writer(
@@ -517,20 +523,19 @@ where
         mut writer: StoreWriter<'_, K, V, D, EP, EL>,
         value: V,
     ) -> Result<bool> {
-        debug_assert!(!writer.inserted);
+        debug_assert!(!writer.is_inserted);
 
         let now = Instant::now();
 
         if !writer.judge() {
-            writer.duration += now.elapsed();
             return Ok(false);
         }
 
-        writer.inserted = true;
+        writer.is_inserted = true;
         let key = &writer.key;
 
         for (i, admission) in self.admissions.iter().enumerate() {
-            let judge = writer.judges.as_ref().unwrap().get(i);
+            let judge = writer.judges.get(i);
             admission.on_insert(key, writer.weight, &self.metrics, judge);
         }
 
@@ -595,17 +600,13 @@ where
     key: K,
     weight: usize,
 
-    /// 1: admit
-    /// 0: reject
-    judges: Option<Judges>,
-    /// 1: use
-    /// 0: ignore
-    mask: Judges,
+    judges: Judges,
+    is_judged: bool,
 
     /// judge duration
     duration: Duration,
 
-    inserted: bool,
+    is_inserted: bool,
 }
 
 impl<'a, K, V, D, EP, EL> StoreWriter<'a, K, V, D, EP, EL>
@@ -621,49 +622,21 @@ where
             store,
             key,
             weight,
-            judges: None,
-            mask: Judges::from_value((1 << store.admissions.len()) - 1),
+            judges: Judges::new(store.admissions.len()),
+            is_judged: false,
             duration: Duration::from_nanos(0),
-            inserted: false,
+            is_inserted: false,
         }
     }
 
-    pub fn set_admission_mask(&mut self, mask: Judges) {
-        self.mask = mask.bitand(Judges::from_value((1 << self.store.admissions.len()) - 1));
-    }
-
-    pub fn all_admission(&mut self) {
-        self.mask = Judges::from_value((1 << self.store.admissions.len()) - 1);
-    }
-
-    pub fn ignore_admission(&mut self) {
-        self.mask = Judges::new();
-    }
-
     /// Judge if the entry can be admitted by configured admission policies.
     pub fn judge(&mut self) -> bool {
         let now = Instant::now();
-        if let Some(judges) = self.judges {
-            self.duration += now.elapsed();
-            return Self::is_admitted(&judges, &self.mask);
+        if !self.is_judged {
+            self.store.judge_inner(self);
         }
-        let judges = self.store.judge_inner(self);
-        self.judges = Some(judges);
         self.duration += now.elapsed();
-        Self::is_admitted(&judges, &self.mask)
-    }
-
-    /// judge | ( ~mask )
-    ///
-    /// | judge | mask | ~mask | result |
-    /// |   0   |  0   |   1   |    1   |
-    /// |   0   |  1   |   0   |    0   |
-    /// |   1   |  0   |   1   |    1   |
-    /// |   1   |  1   |   0   |    1   |
-    fn is_admitted(judges: &Judges, mask: &Judges) -> bool {
-        let mut umask = *mask;
-        umask.invert();
-        judges.bitor(umask).is_full()
+        self.judges.judge()
     }
 
     pub async fn finish(self, value: V) -> Result<bool> {
@@ -683,9 +656,10 @@ where
         f.debug_struct("StoreWriter")
             .field("key", &self.key)
             .field("weight", &self.weight)
-            .field("judged", &self.judges)
+            .field("judges", &self.judges)
+            .field("is_judged", &self.is_judged)
             .field("duration", &self.duration)
-            .field("inserted", &self.inserted)
+            .field("inserted", &self.is_inserted)
             .finish()
     }
 }
@@ -699,15 +673,15 @@ where
     EL: Link,
 {
     fn drop(&mut self) {
-        if !self.inserted {
+        if !self.is_inserted {
             self.store
                 .metrics
                 .latency_insert_dropped
                 .observe(self.duration.as_secs_f64());
             let mut filtered = false;
-            if let Some(judge) = self.judges.as_ref() {
+            if self.is_judged {
                 for (i, admission) in self.store.admissions.iter().enumerate() {
-                    let judge = judge.get(i);
+                    let judge = self.judges.get(i);
                     admission.on_drop(&self.key, self.weight, &self.store.metrics, judge);
                 }
                 filtered = !self.judge();
@@ -871,7 +845,7 @@ fn checksum(buf: &[u8]) -> u64 {
     hasher.finish()
 }
 
-struct RegionEntryIter<K, V, D>
+pub struct RegionEntryIter<K, V, D>
 where
     K: Key,
     V: Value,
@@ -890,7 +864,7 @@ where
     V: Value,
     D: Device,
 {
-    async fn open(region: Region<D>) -> Result<Option<Self>> {
+    pub async fn open(region: Region<D>) -> Result<Option<Self>> {
         let region_size = region.device().region_size();
         let align = region.device().align();
 
@@ -913,7 +887,7 @@ where
         }))
     }
 
-    async fn next(&mut self) -> Result<Option<Index<K>>> {
+    pub async fn next(&mut self) -> Result<Option<Index<K>>> {
         if self.cursor == 0 {
             return Ok(None);
         }
@@ -967,11 +941,28 @@ where
             value_len: footer.value_len,
         }))
     }
+
+    pub async fn next_kv(&mut self) -> Result<Option<(K, V)>> {
+        let index = match self.next().await {
+            Ok(Some(index)) => index,
+            Ok(None) => return Ok(None),
+            Err(e) => return Err(e),
+        };
+
+        // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
+        let start = index.offset as usize;
+        let end = start + index.len as usize;
+        let slice = self.region.load(start..end, 0).await?.unwrap();
+        let kv = read_entry::<K, V>(slice.as_ref());
+        slice.destroy().await;
+
+        Ok(kv)
+    }
 }
 
 #[cfg(test)]
 pub mod tests {
-    use std::path::PathBuf;
+    use std::{collections::HashSet, path::PathBuf};
 
     use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
 
@@ -983,6 +974,113 @@ pub mod tests {
 
     type TestStoreConfig = StoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
+    #[derive(Debug, Clone)]
+    enum Record<K: Key> {
+        Admit(K),
+        Evict(K),
+    }
+
+    #[derive(Debug)]
+    struct JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        records: Mutex<Vec<Record<K>>>,
+        _marker: PhantomData<V>,
+    }
+
+    impl<K, V> JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        fn dump(&self) -> Vec<Record<K>> {
+            self.records.lock().clone()
+        }
+
+        fn remains(&self) -> HashSet<K> {
+            let records = self.dump();
+            let mut res = HashSet::default();
+            for record in records {
+                match record {
+                    Record::Admit(key) => {
+                        res.insert(key);
+                    }
+                    Record::Evict(key) => {
+                        res.remove(&key);
+                    }
+                }
+            }
+            res
+        }
+    }
+
+    impl<K, V> Default for JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        fn default() -> Self {
+            Self {
+                records: Mutex::new(Vec::default()),
+                _marker: PhantomData,
+            }
+        }
+    }
+
+    impl<K, V> AdmissionPolicy for JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        type Key = K;
+
+        type Value = V;
+
+        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+            self.records.lock().push(Record::Admit(key.clone()));
+            true
+        }
+
+        fn on_insert(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+
+        fn on_drop(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+    }
+
+    impl<K, V> ReinsertionPolicy for JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        type Key = K;
+
+        type Value = V;
+
+        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+            self.records.lock().push(Record::Evict(key.clone()));
+            false
+        }
+
+        fn on_insert(
+            &self,
+            _key: &Self::Key,
+            _weight: usize,
+            _metrics: &Arc<crate::metrics::Metrics>,
+            _judge: bool,
+        ) {
+        }
+
+        fn on_drop(
+            &self,
+            _key: &Self::Key,
+            _weight: usize,
+            _metrics: &Arc<crate::metrics::Metrics>,
+            _judge: bool,
+        ) {
+        }
+    }
+
     #[tokio::test]
     #[allow(clippy::identity_op)]
     async fn test_recovery() {
@@ -991,6 +1089,12 @@ pub mod tests {
 
         let tempdir = tempfile::tempdir().unwrap();
 
+        let recorder = Arc::new(JudgeRecorder::default());
+        let admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> =
+            vec![recorder.clone()];
+        let reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> =
+            vec![recorder.clone()];
+
         let config = TestStoreConfig {
             eviction_config: FifoConfig {
                 segment_ratios: vec![1],
@@ -1002,8 +1106,8 @@ pub mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            admissions: vec![],
-            reinsertions: vec![],
+            admissions,
+            reinsertions,
             buffer_pool_size: 8 * MB,
             flushers: 1,
             flush_rate_limit: 0,
@@ -1017,7 +1121,7 @@ pub mod tests {
         let store = TestStore::open(config).await.unwrap();
 
         // files:
-        // [0, 1, 2] (evicted)
+        // [0, 1, 2]
         // [3, 4, 5]
         // [6, 7, 8]
         // [9, 10, 11]
@@ -1025,17 +1129,21 @@ pub mod tests {
             store.insert(i, vec![i as u8; 1 * MB]).await.unwrap();
         }
 
-        for i in 0..3 {
-            assert!(store.lookup(&i).await.unwrap().is_none());
-        }
-        for i in 3..12 {
-            assert_eq!(
-                store.lookup(&i).await.unwrap().unwrap(),
-                vec![i as u8; 1 * MB],
-            );
+        store.shutdown_runners().await.unwrap();
+
+        let remains = recorder.remains();
+
+        for i in 0..12 {
+            if remains.contains(&i) {
+                assert_eq!(
+                    store.lookup(&i).await.unwrap().unwrap(),
+                    vec![i as u8; 1 * MB],
+                );
+            } else {
+                assert!(store.lookup(&i).await.unwrap().is_none());
+            }
         }
 
-        store.shutdown_runners().await.unwrap();
         drop(store);
 
         let config = TestStoreConfig {
@@ -1062,43 +1170,19 @@ pub mod tests {
         };
         let store = TestStore::open(config).await.unwrap();
 
-        for i in 0..3 {
-            assert!(store.lookup(&i).await.unwrap().is_none());
-        }
-        for i in 3..12 {
-            assert_eq!(
-                store.lookup(&i).await.unwrap().unwrap(),
-                vec![i as u8; 1 * MB],
-            );
+        store.shutdown_runners().await.unwrap();
+
+        for i in 0..12 {
+            if remains.contains(&i) {
+                assert_eq!(
+                    store.lookup(&i).await.unwrap().unwrap(),
+                    vec![i as u8; 1 * MB],
+                );
+            } else {
+                assert!(store.lookup(&i).await.unwrap().is_none());
+            }
         }
 
-        store.shutdown_runners().await.unwrap();
         drop(store);
     }
-
-    #[test]
-    fn test_admission_mask() {
-        type T = StoreWriter<
-            'static,
-            u64,
-            Vec<u8>,
-            FsDevice,
-            Fifo<RegionEpItemAdapter<FifoLink>>,
-            FifoLink,
-        >;
-
-        let mask = Judges::from_value(0b_0011);
-
-        assert!(mask.get(0));
-        assert!(mask.get(1));
-        assert!(!mask.get(2));
-        assert!(!mask.get(3));
-
-        let j1 = Judges::from_value(0b_0011);
-        let j2 = Judges::from_value(0b_1011);
-        let j3 = Judges::from_value(0b_1010);
-        assert!(T::is_admitted(&j1, &mask));
-        assert!(T::is_admitted(&j2, &mask));
-        assert!(!T::is_admitted(&j3, &mask));
-    }
 }

From 0b7a363d67eb03483445b160da4ee5806b74db50 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 28 Jul 2023 16:38:46 +0800
Subject: [PATCH 068/261] feat: impl poll mode flushers and reclaimers instead
 of push mode (#90)

* impl poll mode flusher

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* impl poll mode reclaimer

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* clean code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* clean code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/queue.rs           |  29 +++--
 foyer-common/src/rate.rs            |   2 +
 foyer-storage-bench/src/main.rs     |  11 ++
 foyer-storage/src/event.rs          |   4 +-
 foyer-storage/src/flusher.rs        | 149 ++++++---------------
 foyer-storage/src/reclaimer.rs      | 192 ++++++++--------------------
 foyer-storage/src/region_manager.rs | 108 +++++++---------
 foyer-storage/src/store.rs          | 118 +++++++++++------
 8 files changed, 253 insertions(+), 360 deletions(-)

diff --git a/foyer-common/src/queue.rs b/foyer-common/src/queue.rs
index 763ca066..9dd0cb88 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/queue.rs
@@ -12,14 +12,16 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use parking_lot::RwLock;
+use parking_lot::Mutex;
 use std::{collections::VecDeque, fmt::Debug};
-use tokio::sync::Notify;
+use tokio::sync::{watch, Notify};
 
 #[derive(Debug)]
 pub struct AsyncQueue<T> {
-    queue: RwLock<VecDeque<T>>,
+    queue: Mutex<VecDeque<T>>,
     notified: Notify,
+    watch_tx: watch::Sender<usize>,
+    watch_rx: watch::Receiver<usize>,
 }
 
 impl<T: Debug> Default for AsyncQueue<T> {
@@ -30,14 +32,17 @@ impl<T: Debug> Default for AsyncQueue<T> {
 
 impl<T: Debug> AsyncQueue<T> {
     pub fn new() -> Self {
+        let (watch_tx, watch_rx) = watch::channel(0);
         Self {
-            queue: RwLock::new(VecDeque::default()),
+            queue: Mutex::new(VecDeque::default()),
             notified: Notify::new(),
+            watch_tx,
+            watch_rx,
         }
     }
 
     pub fn try_acquire(&self) -> Option<T> {
-        let mut guard = self.queue.write();
+        let mut guard = self.queue.lock();
         if let Some(item) = guard.pop_front() {
             if !guard.is_empty() {
                 // Since in `release` we use `notify_one`, not all waiters
@@ -45,6 +50,7 @@ impl<T: Debug> AsyncQueue<T> {
                 // we call `notify_one` to awake the next pending `acquire`.
                 self.notified.notify_one();
             }
+            self.watch_tx.send(guard.len()).unwrap();
             Some(item)
         } else {
             None
@@ -55,7 +61,7 @@ impl<T: Debug> AsyncQueue<T> {
         loop {
             let notified = self.notified.notified();
             {
-                let mut guard = self.queue.write();
+                let mut guard = self.queue.lock();
                 if let Some(item) = guard.pop_front() {
                     if !guard.is_empty() {
                         // Since in `release` we use `notify_one`, not all waiters
@@ -63,6 +69,7 @@ impl<T: Debug> AsyncQueue<T> {
                         // we call `notify_one` to awake the next pending `acquire`.
                         self.notified.notify_one();
                     }
+                    self.watch_tx.send(guard.len()).unwrap();
                     break item;
                 }
             }
@@ -71,17 +78,23 @@ impl<T: Debug> AsyncQueue<T> {
     }
 
     pub fn release(&self, item: T) {
-        self.queue.write().push_back(item);
+        let mut guard = self.queue.lock();
+        guard.push_back(item);
+        self.watch_tx.send(guard.len()).unwrap();
         self.notified.notify_one();
     }
 
     pub fn len(&self) -> usize {
-        self.queue.read().len()
+        *self.watch_rx.borrow()
     }
 
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
+
+    pub fn watch(&self) -> watch::Receiver<usize> {
+        self.watch_rx.clone()
+    }
 }
 
 #[cfg(test)]
diff --git a/foyer-common/src/rate.rs b/foyer-common/src/rate.rs
index 64f6397f..1dcb0cb8 100644
--- a/foyer-common/src/rate.rs
+++ b/foyer-common/src/rate.rs
@@ -16,11 +16,13 @@ use std::time::{Duration, Instant};
 
 use parking_lot::Mutex;
 
+#[derive(Debug)]
 pub struct RateLimiter {
     inner: Mutex<Inner>,
     rate: f64,
 }
 
+#[derive(Debug)]
 struct Inner {
     quota: f64,
 
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 67372586..dfa31699 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -134,6 +134,10 @@ pub struct Args {
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     reclaim_rate_limit: usize,
+
+    /// `0` means equal to reclaimer count.
+    #[arg(long, default_value_t = 0)]
+    clean_region_threshold: usize,
 }
 
 impl Args {
@@ -243,6 +247,12 @@ async fn main() {
         reinsertions.push(Arc::new(rr));
     }
 
+    let clean_region_threshold = if args.clean_region_threshold == 0 {
+        args.reclaimers
+    } else {
+        args.clean_region_threshold
+    };
+
     let config = StoreConfig {
         eviction_config,
         device_config,
@@ -256,6 +266,7 @@ async fn main() {
         recover_concurrency: args.recover_concurrency,
         event_listeners: vec![],
         prometheus_config: PrometheusConfig::default(),
+        clean_region_threshold,
     };
 
     println!("{:#?}", config);
diff --git a/foyer-storage/src/event.rs b/foyer-storage/src/event.rs
index 0d532c06..0b00f4a7 100644
--- a/foyer-storage/src/event.rs
+++ b/foyer-storage/src/event.rs
@@ -12,6 +12,8 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::fmt::Debug;
+
 use async_trait::async_trait;
 use foyer_common::code::{Key, Value};
 
@@ -19,7 +21,7 @@ use crate::error::Result;
 
 #[allow(unused_variables)]
 #[async_trait]
-pub trait EventListener: Send + Sync + 'static {
+pub trait EventListener: Send + Sync + 'static + Debug {
     type K: Key;
     type V: Value;
 
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index e4771413..7add1da3 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -14,17 +14,14 @@
 
 use std::sync::Arc;
 
-use foyer_common::{queue::AsyncQueue, rate::RateLimiter};
+use foyer_common::rate::RateLimiter;
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use itertools::Itertools;
-use tokio::{
-    sync::{broadcast, mpsc, Mutex},
-    task::JoinHandle,
-};
+
+use tokio::sync::broadcast;
 
 use crate::{
-    device::{BufferAllocator, Device},
-    error::{Error, Result},
+    device::Device,
+    error::Result,
     metrics::Metrics,
     region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
@@ -32,121 +29,47 @@ use crate::{
 };
 
 #[derive(Debug)]
-pub struct FlushTask {
-    pub region_id: RegionId,
-}
-
-struct FlusherInner {
-    sequence: usize,
-
-    task_txs: Vec<mpsc::UnboundedSender<FlushTask>>,
-}
-
-pub struct Flusher {
-    runners: usize,
-
-    inner: Mutex<FlusherInner>,
-}
-
-impl Flusher {
-    pub fn new(runners: usize) -> Self {
-        let inner = FlusherInner {
-            sequence: 0,
-            task_txs: Vec::with_capacity(runners),
-        };
-        Self {
-            runners,
-            inner: Mutex::new(inner),
-        }
-    }
-
-    pub async fn run<D, E, EL>(
-        &self,
-        buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
-        region_manager: Arc<RegionManager<D, E, EL>>,
-        rate_limiter: Option<Arc<RateLimiter>>,
-        stop_rxs: Vec<broadcast::Receiver<()>>,
-        metrics: Arc<Metrics>,
-    ) -> Vec<JoinHandle<()>>
-    where
-        D: Device,
-        E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-        EL: Link,
-    {
-        let mut inner = self.inner.lock().await;
-
-        #[allow(clippy::type_complexity)]
-        let (mut txs, rxs): (
-            Vec<mpsc::UnboundedSender<FlushTask>>,
-            Vec<mpsc::UnboundedReceiver<FlushTask>>,
-        ) = (0..self.runners).map(|_| mpsc::unbounded_channel()).unzip();
-        inner.task_txs.append(&mut txs);
-
-        let runners = rxs
-            .into_iter()
-            .zip_eq(stop_rxs.into_iter())
-            .map(|(task_rx, stop_rx)| Runner {
-                task_rx,
-                buffers: buffers.clone(),
-                region_manager: region_manager.clone(),
-                rate_limiter: rate_limiter.clone(),
-                stop_rx,
-                metrics: metrics.clone(),
-            })
-            .collect_vec();
-
-        let mut handles = vec![];
-        for runner in runners {
-            let handle = tokio::spawn(async move {
-                runner.run().await.unwrap();
-            });
-            handles.push(handle);
-        }
-        handles
-    }
-
-    pub fn runners(&self) -> usize {
-        self.runners
-    }
-
-    pub async fn submit(&self, task: FlushTask) -> Result<()> {
-        let mut inner = self.inner.lock().await;
-        let submittee = inner.sequence % inner.task_txs.len();
-        inner.sequence += 1;
-        inner.task_txs[submittee].send(task).map_err(Error::other)
-    }
-}
-
-struct Runner<D, E, EL>
+pub struct Flusher<D, EP, EL>
 where
     D: Device,
-    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    task_rx: mpsc::UnboundedReceiver<FlushTask>,
-    buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
-
-    region_manager: Arc<RegionManager<D, E, EL>>,
+    region_manager: Arc<RegionManager<D, EP, EL>>,
 
     rate_limiter: Option<Arc<RateLimiter>>,
 
-    stop_rx: broadcast::Receiver<()>,
-
     metrics: Arc<Metrics>,
+
+    stop_rx: broadcast::Receiver<()>,
 }
 
-impl<D, E, EL> Runner<D, E, EL>
+impl<D, EP, EL> Flusher<D, EP, EL>
 where
     D: Device,
-    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    async fn run(mut self) -> Result<()> {
+    pub fn new(
+        region_manager: Arc<RegionManager<D, EP, EL>>,
+        rate_limiter: Option<Arc<RateLimiter>>,
+        metrics: Arc<Metrics>,
+        stop_rx: broadcast::Receiver<()>,
+    ) -> Self {
+        Self {
+            region_manager,
+            rate_limiter,
+            metrics,
+            stop_rx,
+        }
+    }
+
+    pub async fn run(mut self) -> Result<()> {
         loop {
             tokio::select! {
                 biased;
-                Some(task) = self.task_rx.recv() => {
-                    self.handle(task).await?;
+                region_id = self.region_manager.dirty_regions().acquire() => {
+                    self.handle(region_id).await?;
                 }
                 _ = self.stop_rx.recv() => {
                     tracing::info!("[flusher] exit");
@@ -156,10 +79,10 @@ where
         }
     }
 
-    async fn handle(&self, task: FlushTask) -> Result<()> {
-        tracing::info!("[flusher] receive flush task, region: {}", task.region_id);
+    async fn handle(&self, region_id: RegionId) -> Result<()> {
+        tracing::info!("[flusher] receive flush task, region: {}", region_id);
 
-        let region = self.region_manager.region(&task.region_id);
+        let region = self.region_manager.region(&region_id);
 
         tracing::trace!("[flusher] step 1");
 
@@ -171,7 +94,7 @@ where
             let _ = region.exclusive(false, true, false).await;
         }
 
-        tracing::trace!("[flusher] write region {} back to device", task.region_id);
+        tracing::trace!("[flusher] write region {} back to device", region_id);
 
         let mut offset = 0;
         let len = region.device().io_size();
@@ -202,10 +125,10 @@ where
         tracing::trace!("[flusher] step 3");
 
         // step 3: release buffer
-        self.buffers.release(buffer);
-        self.region_manager.set_region_evictable(&region.id()).await;
+        self.region_manager.buffers().release(buffer);
+        self.region_manager.eviction_push(region.id());
 
-        tracing::info!("[flusher] finish flush task, region: {}", task.region_id);
+        tracing::info!("[flusher] finish flush task, region: {}", region_id);
 
         self.metrics
             .bytes_flush
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index caa9148a..d53bf575 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -12,135 +12,27 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 
 use crate::{
     device::Device,
-    error::{Error, Result},
+    error::Result,
     event::EventListener,
-    indices::Indices,
     judge::Judges,
     metrics::Metrics,
-    region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
-    reinsertion::ReinsertionPolicy,
     store::{RegionEntryIter, Store},
 };
 use bytes::BufMut;
 use foyer_common::{
     code::{Key, Value},
-    queue::AsyncQueue,
     rate::RateLimiter,
 };
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use itertools::Itertools;
-use tokio::{
-    sync::{broadcast, mpsc, Mutex},
-    task::JoinHandle,
-};
+use tokio::sync::broadcast;
 
 #[derive(Debug)]
-pub struct ReclaimTask {
-    pub region_id: RegionId,
-}
-
-struct ReclaimerInner {
-    sequence: usize,
-
-    task_txs: Vec<mpsc::Sender<ReclaimTask>>,
-}
-
-pub struct Reclaimer {
-    runners: usize,
-
-    inner: Mutex<ReclaimerInner>,
-}
-
-impl Reclaimer {
-    pub fn new(runners: usize) -> Self {
-        let inner = ReclaimerInner {
-            sequence: 0,
-            task_txs: Vec::with_capacity(runners),
-        };
-
-        Self {
-            runners,
-            inner: Mutex::new(inner),
-        }
-    }
-
-    #[allow(clippy::too_many_arguments)]
-    pub async fn run<K, V, D, EP, EL>(
-        &self,
-        store: Arc<Store<K, V, D, EP, EL>>,
-        region_manager: Arc<RegionManager<D, EP, EL>>,
-        clean_regions: Arc<AsyncQueue<RegionId>>,
-        reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
-        indices: Arc<Indices<K>>,
-        rate_limiter: Option<Arc<RateLimiter>>,
-        event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
-        stop_rxs: Vec<broadcast::Receiver<()>>,
-        metrics: Arc<Metrics>,
-    ) -> Vec<JoinHandle<()>>
-    where
-        K: Key,
-        V: Value,
-        D: Device,
-        EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-        EL: Link,
-    {
-        let mut inner = self.inner.lock().await;
-
-        #[allow(clippy::type_complexity)]
-        let (mut txs, rxs): (
-            Vec<mpsc::Sender<ReclaimTask>>,
-            Vec<mpsc::Receiver<ReclaimTask>>,
-        ) = (0..self.runners).map(|_| mpsc::channel(1)).unzip();
-        inner.task_txs.append(&mut txs);
-
-        let runners = rxs
-            .into_iter()
-            .zip_eq(stop_rxs.into_iter())
-            .map(|(task_rx, stop_rx)| Runner {
-                task_rx,
-                store: store.clone(),
-                region_manager: region_manager.clone(),
-                clean_regions: clean_regions.clone(),
-                reinsertions: reinsertions.clone(),
-                indices: indices.clone(),
-                rate_limiter: rate_limiter.clone(),
-                stop_rx,
-                metrics: metrics.clone(),
-                event_listeners: event_listeners.clone(),
-            })
-            .collect_vec();
-
-        let mut handles = vec![];
-        for runner in runners {
-            let handle = tokio::spawn(async move {
-                runner.run().await.unwrap();
-            });
-            handles.push(handle);
-        }
-        handles
-    }
-
-    pub fn runners(&self) -> usize {
-        self.runners
-    }
-
-    pub async fn submit(&self, task: ReclaimTask) -> Result<()> {
-        let mut inner = self.inner.lock().await;
-        let submittee = inner.sequence % inner.task_txs.len();
-        inner.sequence += 1;
-        inner.task_txs[submittee]
-            .send(task)
-            .await
-            .map_err(Error::other)
-    }
-}
-
-struct Runner<K, V, D, EP, EL>
+pub struct Reclaimer<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -148,24 +40,22 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    task_rx: mpsc::Receiver<ReclaimTask>,
+    threshold: usize,
 
     store: Arc<Store<K, V, D, EP, EL>>,
+
     region_manager: Arc<RegionManager<D, EP, EL>>,
-    clean_regions: Arc<AsyncQueue<RegionId>>,
-    reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
-    indices: Arc<Indices<K>>,
 
     rate_limiter: Option<Arc<RateLimiter>>,
 
     event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
 
-    stop_rx: broadcast::Receiver<()>,
-
     metrics: Arc<Metrics>,
+
+    stop_rx: broadcast::Receiver<()>,
 }
 
-impl<K, V, D, EP, EL> Runner<K, V, D, EP, EL>
+impl<K, V, D, EP, EL> Reclaimer<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -173,12 +63,33 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    async fn run(mut self) -> Result<()> {
+    pub fn new(
+        threshold: usize,
+        store: Arc<Store<K, V, D, EP, EL>>,
+        region_manager: Arc<RegionManager<D, EP, EL>>,
+        rate_limiter: Option<Arc<RateLimiter>>,
+        event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
+        metrics: Arc<Metrics>,
+        stop_rx: broadcast::Receiver<()>,
+    ) -> Self {
+        Self {
+            threshold,
+            store,
+            region_manager,
+            rate_limiter,
+            event_listeners,
+            metrics,
+            stop_rx,
+        }
+    }
+
+    pub async fn run(mut self) -> Result<()> {
+        let mut watch = self.region_manager.clean_regions().watch();
         loop {
             tokio::select! {
                 biased;
-                Some(task) = self.task_rx.recv() => {
-                    self.handle(task).await?;
+                Ok(()) = watch.changed() => {
+                    self.handle().await?;
                 }
                 _ = self.stop_rx.recv() => {
                     tracing::info!("[reclaimer] exit");
@@ -188,16 +99,22 @@ where
         }
     }
 
-    async fn handle(&self, task: ReclaimTask) -> Result<()> {
-        tracing::info!(
-            "[reclaimer] receive reclaim task, region: {}",
-            task.region_id
-        );
+    async fn handle(&self) -> Result<()> {
+        if self.region_manager.clean_regions().len() >= self.threshold {
+            return Ok(());
+        }
 
-        let region = self.region_manager.region(&task.region_id);
+        // TODO(MrCroxx): subscribe evictable region changes.
+        let region_id = loop {
+            match self.region_manager.eviction_pop() {
+                Some(id) => break id,
+                None => tokio::time::sleep(Duration::from_millis(100)).await,
+            }
+        };
+        let region = self.region_manager.region(&region_id);
 
         // step 1: drop indices
-        let indices = self.indices.take_region(&task.region_id);
+        let indices = self.store.indices().take_region(&region_id);
         for index in indices.iter() {
             for listener in self.event_listeners.iter() {
                 listener.on_evict(&index.key).await?;
@@ -222,9 +139,9 @@ where
             let region = region.clone();
             let metrics = self.metrics.clone();
             let rate = self.rate_limiter.clone();
-            let reinsertions = self.reinsertions.clone();
+            let reinsertions = self.store.reinsertions().clone();
 
-            tracing::info!("[reclaimer] begin reinsertion, region: {}", task.region_id);
+            tracing::info!("[reclaimer] begin reinsertion, region: {}", region_id);
 
             async move {
                 let mut iter = match RegionEntryIter::<K, V, D>::open(region).await {
@@ -268,13 +185,13 @@ where
                     metrics.bytes_reinsert.inc_by(weight as u64);
                 }
 
-                tracing::info!("[reclaimer] finish reinsertion, region: {}", task.region_id);
+                tracing::info!("[reclaimer] finish reinsertion, region: {}", region_id);
 
                 Ok(())
             }
         };
 
-        if  !self.reinsertions.is_empty() && let Err(e) = reinsert().await {
+        if  !self.store.reinsertions().is_empty() && let Err(e) = reinsert().await {
             tracing::warn!("reinsert region {:?} error: {:?}", region, e);
         }
 
@@ -285,16 +202,13 @@ where
         (&mut buf[..]).put_slice(&vec![0; align]);
         region
             .device()
-            .write(buf, task.region_id, (region_size - align) as u64, align)
+            .write(buf, region_id, (region_size - align) as u64, align)
             .await?;
 
         // step 4: send clean region
-        self.clean_regions.release(task.region_id);
+        self.region_manager.clean_regions().release(region_id);
 
-        tracing::info!(
-            "[reclaimer] finish reclaim task, region: {}",
-            task.region_id
-        );
+        tracing::info!("[reclaimer] finish reclaim task, region: {}", region_id);
 
         self.metrics
             .bytes_reclaim
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index f9ef5a21..6d55796d 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -25,8 +25,6 @@ use tokio::sync::RwLock as AsyncRwLock;
 
 use crate::{
     device::Device,
-    flusher::{FlushTask, Flusher},
-    reclaimer::{ReclaimTask, Reclaimer},
     region::{AllocateResult, Region, RegionId},
 };
 
@@ -44,51 +42,66 @@ intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEp
 key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
 priority_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { priority: usize } where L: Link }
 
+#[derive(Debug)]
 struct RegionManagerInner {
     current: Option<RegionId>,
 }
 
-pub struct RegionManager<D, E, EL>
+/// Manager of regions and buffer pools.
+///
+/// # Region Lifetime
+///
+/// `clean` ==(allocate)=> `dirty` ==(flush)=> `evictable` ==(reclaim)=> `clean`
+#[derive(Debug)]
+pub struct RegionManager<D, EP, EL>
 where
     D: Device,
-    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     inner: Arc<AsyncRwLock<RegionManagerInner>>,
 
     buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
+
+    /// Empty regions.
     clean_regions: Arc<AsyncQueue<RegionId>>,
 
+    /// Regions with dirty buffer waiting for flushing.
+    dirty_regions: Arc<AsyncQueue<RegionId>>,
+
     regions: Vec<Region<D>>,
     items: Vec<Arc<RegionEpItem<EL>>>,
 
-    flusher: Arc<Flusher>,
-    reclaimer: Arc<Reclaimer>,
-
-    eviction: RwLock<E>,
+    eviction: RwLock<EP>,
 }
 
-impl<D, E, EL> RegionManager<D, E, EL>
+impl<D, EP, EL> RegionManager<D, EP, EL>
 where
     D: Device,
-    E: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     pub fn new(
-        region_nums: usize,
-        eviction_config: E::Config,
-        buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
-        clean_regions: Arc<AsyncQueue<RegionId>>,
+        buffer_count: usize,
+        region_count: usize,
+        eviction_config: EP::Config,
         device: D,
-        flusher: Arc<Flusher>,
-        reclaimer: Arc<Reclaimer>,
     ) -> Self {
-        let eviction = E::new(eviction_config);
+        let buffers = Arc::new(AsyncQueue::new());
+        for _ in 0..buffer_count {
+            let len = device.region_size();
+            let buffer = device.io_buffer(len, len);
+            buffers.release(buffer);
+        }
+
+        let eviction = EP::new(eviction_config);
+        let clean_regions = Arc::new(AsyncQueue::new());
+        let dirty_regions = Arc::new(AsyncQueue::new());
 
-        let mut regions = Vec::with_capacity(region_nums);
-        let mut items = Vec::with_capacity(region_nums);
+        let mut regions = Vec::with_capacity(region_count);
+        let mut items = Vec::with_capacity(region_count);
 
-        for id in 0..region_nums as RegionId {
+        for id in 0..region_count as RegionId {
             let region = Region::new(id, device.clone());
             let item = Arc::new(RegionEpItem {
                 link: EL::default(),
@@ -106,10 +119,9 @@ where
             inner: Arc::new(AsyncRwLock::new(inner)),
             buffers,
             clean_regions,
+            dirty_regions,
             regions,
             items,
-            flusher,
-            reclaimer,
             eviction: RwLock::new(eviction),
         }
     }
@@ -127,8 +139,8 @@ where
                     return AllocateResult::Ok(slice);
                 }
                 AllocateResult::Full { slice, remain } => {
-                    // current region is full, schedule flushing
-                    self.submit_flush_task(FlushTask { region_id }).await;
+                    // current region is full, append dirty regions
+                    self.dirty_regions.release(region_id);
                     inner.current = None;
                     return AllocateResult::Full { slice, remain };
                 }
@@ -137,15 +149,6 @@ where
 
         assert!(inner.current.is_none());
 
-        tracing::debug!("clean regions: {}", self.clean_regions.len());
-        if self.clean_regions.len() < self.reclaimer.runners() {
-            let item = self.eviction.write().pop();
-            if let Some(item) = item {
-                self.submit_reclaim_task(ReclaimTask { region_id: item.id })
-                    .await;
-            }
-        }
-
         let region_id = self.clean_regions.acquire().await;
         tracing::info!("switch to clean region: {}", region_id);
 
@@ -176,40 +179,25 @@ where
         }
     }
 
-    #[tracing::instrument(skip(self))]
-    pub async fn set_region_evictable(&self, id: &RegionId) {
-        let to_reclaim = {
-            let mut eviction = self.eviction.write();
-            let item = &self.items[*id as usize];
-            if !item.link.is_linked() {
-                eviction.push(item.clone());
-            }
-            if self.clean_regions.len() < self.reclaimer.runners() {
-                Some(eviction.pop().unwrap())
-            } else {
-                None
-            }
-        };
-
-        if let Some(item) = to_reclaim {
-            self.submit_reclaim_task(ReclaimTask { region_id: item.id })
-                .await
-        }
+    pub fn buffers(&self) -> &AsyncQueue<Vec<u8, D::IoBufferAllocator>> {
+        &self.buffers
     }
 
     pub fn clean_regions(&self) -> &AsyncQueue<RegionId> {
         &self.clean_regions
     }
 
-    async fn submit_reclaim_task(&self, task: ReclaimTask) {
-        if let Err(e) = self.reclaimer.submit(task).await {
-            tracing::warn!("fail to submit reclaim task: {:?}", e);
-        }
+    pub fn dirty_regions(&self) -> &AsyncQueue<RegionId> {
+        &self.dirty_regions
     }
 
-    async fn submit_flush_task(&self, task: FlushTask) {
-        if let Err(e) = self.flusher.submit(task).await {
-            tracing::warn!("fail to submit reclaim task: {:?}", e);
-        }
+    pub fn eviction_push(&self, region_id: RegionId) {
+        self.eviction
+            .write()
+            .push(self.items[region_id as usize].clone());
+    }
+
+    pub fn eviction_pop(&self) -> Option<RegionId> {
+        self.eviction.write().pop().map(|item| item.id)
     }
 }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 056b4dd4..4de9bdd1 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -20,7 +20,7 @@ use std::{
 };
 
 use bytes::{Buf, BufMut};
-use foyer_common::{bits, queue::AsyncQueue, rate::RateLimiter};
+use foyer_common::{bits, rate::RateLimiter};
 use foyer_intrusive::eviction::EvictionPolicy;
 use futures::Future;
 use itertools::Itertools;
@@ -63,17 +63,45 @@ where
     D: Device,
     EP: EvictionPolicy,
 {
+    /// Evictino policy configurations.
     pub eviction_config: EP::Config,
+
+    /// Device configurations.
     pub device_config: D::Config,
+
+    /// Admission policies.
     pub admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
+
+    /// Reinsertion policies.
     pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
+
+    /// Buffer pool size, should be a multiplier of device region size.
     pub buffer_pool_size: usize,
+
+    /// Count of flushers.
     pub flushers: usize,
+
+    /// Flush rate limits.
     pub flush_rate_limit: usize,
+
+    /// Count of reclaimers.
     pub reclaimers: usize,
+
+    /// Flush rate limits.
     pub reclaim_rate_limit: usize,
+
+    /// Clean region count threshold to trigger reclamation.
+    ///
+    /// `clean_region_threshold` is recommended to be equal or larger than `reclaimers`.
+    pub clean_region_threshold: usize,
+
+    /// Concurrency of recovery.
     pub recover_concurrency: usize,
+
+    /// Event listsners.
     pub event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
+
+    /// Prometheus configuration.
     pub prometheus_config: PrometheusConfig,
 }
 
@@ -97,6 +125,7 @@ where
     }
 }
 
+#[derive(Debug)]
 pub struct Store<K, V, D, EP, EL>
 where
     K: Key,
@@ -112,6 +141,7 @@ where
     device: D,
 
     admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
+    reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
 
     event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
 
@@ -137,26 +167,13 @@ where
 
         let device = D::open(config.device_config).await?;
 
-        let buffers = Arc::new(AsyncQueue::new());
-        for _ in 0..(config.buffer_pool_size / device.region_size()) {
-            let len = device.region_size();
-            let buffer = device.io_buffer(len, len);
-            buffers.release(buffer);
-        }
-
-        let clean_regions = Arc::new(AsyncQueue::new());
-
-        let flusher = Arc::new(Flusher::new(config.flushers));
-        let reclaimer = Arc::new(Reclaimer::new(config.reclaimers));
+        let buffer_count = config.buffer_pool_size / device.region_size();
 
         let region_manager = Arc::new(RegionManager::new(
+            buffer_count,
             device.regions(),
             config.eviction_config,
-            buffers.clone(),
-            clean_regions.clone(),
             device.clone(),
-            flusher.clone(),
-            reclaimer.clone(),
         ));
 
         let indices = Arc::new(Indices::new(device.regions()));
@@ -187,6 +204,7 @@ where
             region_manager: region_manager.clone(),
             device: device.clone(),
             admissions: config.admissions,
+            reinsertions: config.reinsertions,
             event_listeners: config.event_listeners.clone(),
             handles: Mutex::new(vec![]),
             stop_tx,
@@ -197,7 +215,7 @@ where
         for admission in store.admissions.iter() {
             admission.init(&store.indices);
         }
-        for reinsertion in config.reinsertions.iter() {
+        for reinsertion in store.reinsertions.iter() {
             reinsertion.init(&store.indices);
         }
 
@@ -210,32 +228,44 @@ where
             rate => Some(Arc::new(RateLimiter::new(rate as f64))),
         };
 
-        let mut handles = vec![];
-        handles.append(
-            &mut flusher
-                .run(
-                    buffers,
+        let flushers = flusher_stop_rxs
+            .into_iter()
+            .map(|stop_rx| {
+                Flusher::new(
                     region_manager.clone(),
-                    flush_rate_limiter,
-                    flusher_stop_rxs,
+                    flush_rate_limiter.clone(),
                     metrics.clone(),
+                    stop_rx,
                 )
-                .await,
-        );
-        handles.append(
-            &mut reclaimer
-                .run(
+            })
+            .collect_vec();
+        let reclaimers = reclaimer_stop_rxs
+            .into_iter()
+            .map(|stop_rx| {
+                Reclaimer::new(
+                    config.clean_region_threshold,
                     store.clone(),
-                    region_manager,
-                    clean_regions,
-                    config.reinsertions,
-                    indices,
-                    reclaim_rate_limiter,
-                    config.event_listeners,
-                    reclaimer_stop_rxs,
-                    metrics,
+                    region_manager.clone(),
+                    reclaim_rate_limiter.clone(),
+                    config.event_listeners.clone(),
+                    metrics.clone(),
+                    stop_rx,
                 )
-                .await,
+            })
+            .collect_vec();
+
+        let mut handles = vec![];
+        handles.append(
+            &mut flushers
+                .into_iter()
+                .map(|flusher| tokio::spawn(async move { flusher.run().await.unwrap() }))
+                .collect_vec(),
+        );
+        handles.append(
+            &mut reclaimers
+                .into_iter()
+                .map(|reclaimer| tokio::spawn(async move { reclaimer.run().await.unwrap() }))
+                .collect_vec(),
         );
         store.handles.lock().append(&mut handles);
 
@@ -421,6 +451,14 @@ where
         Ok(())
     }
 
+    pub(crate) fn indices(&self) -> &Arc<Indices<K>> {
+        &self.indices
+    }
+
+    pub(crate) fn reinsertions(&self) -> &Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>> {
+        &self.reinsertions
+    }
+
     fn serialized_len(&self, key: &K, value: &V) -> usize {
         let unaligned =
             key.serialized_len() + value.serialized_len() + EntryFooter::serialized_len();
@@ -501,7 +539,7 @@ where
                 }
                 indices.insert(index);
             }
-            region_manager.set_region_evictable(&region_id).await;
+            region_manager.eviction_push(region_id);
             true
         } else {
             region_manager.clean_regions().release(region_id);
@@ -1116,6 +1154,7 @@ pub mod tests {
             recover_concurrency: 2,
             event_listeners: vec![],
             prometheus_config: PrometheusConfig::default(),
+            clean_region_threshold: 1,
         };
 
         let store = TestStore::open(config).await.unwrap();
@@ -1167,6 +1206,7 @@ pub mod tests {
             recover_concurrency: 2,
             event_listeners: vec![],
             prometheus_config: PrometheusConfig::default(),
+            clean_region_threshold: 1,
         };
         let store = TestStore::open(config).await.unwrap();
 

From 7f19bf80846ce51ed73e17699c00dca0e6710e2c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 29 Jul 2023 12:59:39 +0800
Subject: [PATCH 069/261] fix: resolve reinsertion stack by modify allocation
 (#91)

* fix: resolve reinsertion stack by modify allocation

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* remove unnecessary Arc

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* clean code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/batch.rs           | 72 ++++++++++++++++++++++
 foyer-common/src/lib.rs             |  1 +
 foyer-common/src/queue.rs           |  4 ++
 foyer-storage/src/reclaimer.rs      |  5 +-
 foyer-storage/src/region.rs         |  2 +
 foyer-storage/src/region_manager.rs | 93 ++++++++++++++++++-----------
 foyer-storage/src/store.rs          | 54 +++++++++++------
 7 files changed, 176 insertions(+), 55 deletions(-)
 create mode 100644 foyer-common/src/batch.rs

diff --git a/foyer-common/src/batch.rs b/foyer-common/src/batch.rs
new file mode 100644
index 00000000..9d2d6b79
--- /dev/null
+++ b/foyer-common/src/batch.rs
@@ -0,0 +1,72 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use parking_lot::Mutex;
+use tokio::sync::oneshot;
+
+const DEFAULT_CAPACITY: usize = 16;
+
+pub enum Identity<T> {
+    Leader(oneshot::Receiver<T>),
+    Follower(oneshot::Receiver<T>),
+}
+
+#[derive(Debug)]
+pub struct Item<A, T> {
+    pub arg: A,
+    pub tx: oneshot::Sender<T>,
+}
+
+#[derive(Debug)]
+pub struct Batch<A, T> {
+    queue: Mutex<Vec<Item<A, T>>>,
+}
+
+impl<A, T> Default for Batch<A, T> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<A, T> Batch<A, T> {
+    pub fn new() -> Self {
+        Self::with_capacity(DEFAULT_CAPACITY)
+    }
+
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            queue: Mutex::new(Vec::with_capacity(capacity)),
+        }
+    }
+
+    pub fn push(&self, arg: A) -> Identity<T> {
+        let (tx, rx) = oneshot::channel();
+        let item = Item { arg, tx };
+        let mut queue = self.queue.lock();
+        let is_leader = queue.is_empty();
+        queue.push(item);
+        if is_leader {
+            Identity::Leader(rx)
+        } else {
+            Identity::Follower(rx)
+        }
+    }
+
+    pub fn rotate(&self) -> Vec<Item<A, T>> {
+        let mut queue = self.queue.lock();
+        let mut q = Vec::with_capacity(queue.capacity());
+        std::mem::swap(&mut *queue, &mut q);
+        q
+    }
+}
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 02f0e9d6..1aa9d2cc 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -14,6 +14,7 @@
 
 #![feature(trait_alias)]
 
+pub mod batch;
 pub mod bits;
 pub mod code;
 pub mod queue;
diff --git a/foyer-common/src/queue.rs b/foyer-common/src/queue.rs
index 9dd0cb88..4ab33e7e 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/queue.rs
@@ -95,6 +95,10 @@ impl<T: Debug> AsyncQueue<T> {
     pub fn watch(&self) -> watch::Receiver<usize> {
         self.watch_rx.clone()
     }
+
+    pub fn flash(&self) {
+        self.watch_tx.send(self.len()).unwrap();
+    }
 }
 
 #[cfg(test)]
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index d53bf575..edb087f4 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -170,7 +170,10 @@ where
                         tokio::time::sleep(wait).await;
                     }
 
-                    if self.store.insert(key.clone(), value).await? {
+                    let mut writer = self.store.writer(key.clone(), weight);
+                    writer.set_skippable();
+
+                    if writer.finish(value).await? {
                         for (index, reinsertion) in reinsertions.iter().enumerate() {
                             let judge = judges.get(index);
                             reinsertion.on_insert(&key, weight, &metrics, judge);
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index c6bf62dd..1e8240c8 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -31,6 +31,7 @@ pub type Version = u32;
 pub enum AllocateResult {
     Ok(WriteSlice),
     Full { slice: WriteSlice, remain: usize },
+    None,
 }
 
 impl AllocateResult {
@@ -38,6 +39,7 @@ impl AllocateResult {
         match self {
             AllocateResult::Ok(slice) => slice,
             AllocateResult::Full { .. } => unreachable!(),
+            AllocateResult::None => unreachable!(),
         }
     }
 }
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 6d55796d..10618817 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -14,7 +14,10 @@
 
 use std::sync::Arc;
 
-use foyer_common::queue::AsyncQueue;
+use foyer_common::{
+    batch::{Batch, Identity},
+    queue::AsyncQueue,
+};
 use foyer_intrusive::{
     core::adapter::Link,
     eviction::{EvictionPolicy, EvictionPolicyExt},
@@ -42,11 +45,6 @@ intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEp
 key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
 priority_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { priority: usize } where L: Link }
 
-#[derive(Debug)]
-struct RegionManagerInner {
-    current: Option<RegionId>,
-}
-
 /// Manager of regions and buffer pools.
 ///
 /// # Region Lifetime
@@ -59,19 +57,22 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    inner: Arc<AsyncRwLock<RegionManagerInner>>,
+    current: AsyncRwLock<Option<RegionId>>,
+    rotate_batch: Batch<(), ()>,
 
-    buffers: Arc<AsyncQueue<Vec<u8, D::IoBufferAllocator>>>,
+    /// Buffer pool for dirty buffers.
+    buffers: AsyncQueue<Vec<u8, D::IoBufferAllocator>>,
 
     /// Empty regions.
-    clean_regions: Arc<AsyncQueue<RegionId>>,
+    clean_regions: AsyncQueue<RegionId>,
 
     /// Regions with dirty buffer waiting for flushing.
-    dirty_regions: Arc<AsyncQueue<RegionId>>,
+    dirty_regions: AsyncQueue<RegionId>,
 
     regions: Vec<Region<D>>,
     items: Vec<Arc<RegionEpItem<EL>>>,
 
+    /// Eviction policy.
     eviction: RwLock<EP>,
 }
 
@@ -87,7 +88,7 @@ where
         eviction_config: EP::Config,
         device: D,
     ) -> Self {
-        let buffers = Arc::new(AsyncQueue::new());
+        let buffers = AsyncQueue::new();
         for _ in 0..buffer_count {
             let len = device.region_size();
             let buffer = device.io_buffer(len, len);
@@ -95,8 +96,8 @@ where
         }
 
         let eviction = EP::new(eviction_config);
-        let clean_regions = Arc::new(AsyncQueue::new());
-        let dirty_regions = Arc::new(AsyncQueue::new());
+        let clean_regions = AsyncQueue::new();
+        let dirty_regions = AsyncQueue::new();
 
         let mut regions = Vec::with_capacity(region_count);
         let mut items = Vec::with_capacity(region_count);
@@ -113,10 +114,9 @@ where
             items.push(item);
         }
 
-        let inner = RegionManagerInner { current: None };
-
         Self {
-            inner: Arc::new(AsyncRwLock::new(inner)),
+            current: AsyncRwLock::new(None),
+            rotate_batch: Batch::new(),
             buffers,
             clean_regions,
             dirty_regions,
@@ -128,41 +128,62 @@ where
 
     /// Allocate a buffer slice with given size in an active region to write.
     #[tracing::instrument(skip(self))]
-    pub async fn allocate(&self, size: usize) -> AllocateResult {
-        let mut inner = self.inner.write().await;
+    pub async fn allocate(&self, size: usize, must_allocate: bool) -> AllocateResult {
+        loop {
+            let res = self.allocate_inner(size).await;
+
+            if !must_allocate || !matches!(res, AllocateResult::None) {
+                return res;
+            }
+
+            self.rotate().await;
+        }
+    }
 
-        // try allocate from current region
-        if let Some(region_id) = inner.current {
+    pub async fn allocate_inner(&self, size: usize) -> AllocateResult {
+        let mut current = self.current.write().await;
+        if let Some(region_id) = *current {
             let region = self.region(&region_id);
             match region.allocate(size).await {
-                AllocateResult::Ok(slice) => {
-                    return AllocateResult::Ok(slice);
-                }
+                AllocateResult::Ok(slice) => AllocateResult::Ok(slice),
                 AllocateResult::Full { slice, remain } => {
                     // current region is full, append dirty regions
                     self.dirty_regions.release(region_id);
-                    inner.current = None;
-                    return AllocateResult::Full { slice, remain };
+                    *current = None;
+                    AllocateResult::Full { slice, remain }
                 }
+                AllocateResult::None => unreachable!(),
             }
+        } else {
+            AllocateResult::None
         }
+    }
 
-        assert!(inner.current.is_none());
-
-        let region_id = self.clean_regions.acquire().await;
-        tracing::info!("switch to clean region: {}", region_id);
+    pub async fn rotate(&self) {
+        match self.rotate_batch.push(()) {
+            Identity::Leader(rx) => {
+                // Wait a clean region to be released.
+                let region_id = self.clean_regions.acquire().await;
 
-        let region = self.region(&region_id);
-        region.advance().await;
+                tracing::info!("switch to clean region: {}", region_id);
 
-        let buffer = self.buffers.acquire().await;
-        region.attach_buffer(buffer).await;
+                let region = self.region(&region_id);
+                region.advance().await;
 
-        let slice = region.allocate(size).await.unwrap();
+                let buffer = self.buffers.acquire().await;
+                region.attach_buffer(buffer).await;
 
-        inner.current = Some(region_id);
+                *self.current.write().await = Some(region_id);
 
-        AllocateResult::Ok(slice)
+                // Notify the batch to advance.
+                let items = self.rotate_batch.rotate();
+                for item in items {
+                    item.tx.send(()).unwrap();
+                }
+                rx.await.unwrap();
+            }
+            Identity::Follower(rx) => rx.await.unwrap(),
+        }
     }
 
     #[tracing::instrument(skip(self))]
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 4de9bdd1..0bf6de19 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -38,7 +38,7 @@ use crate::{
     judge::Judges,
     metrics::Metrics,
     reclaimer::Reclaimer,
-    region::{Region, RegionId},
+    region::{AllocateResult, Region, RegionId},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
 };
@@ -254,6 +254,8 @@ where
             })
             .collect_vec();
 
+        store.recover(config.recover_concurrency).await?;
+
         let mut handles = vec![];
         handles.append(
             &mut flushers
@@ -269,8 +271,6 @@ where
         );
         store.handles.lock().append(&mut handles);
 
-        store.recover(config.recover_concurrency).await?;
-
         Ok(store)
     }
 
@@ -468,10 +468,10 @@ where
     async fn seal(&self) -> Result<()> {
         match self
             .region_manager
-            .allocate(self.device.region_size() - self.device.align())
+            .allocate(self.device.region_size() - self.device.align(), true)
             .await
         {
-            crate::region::AllocateResult::Full { mut slice, remain } => {
+            AllocateResult::Full { mut slice, remain } => {
                 // current region is full, write region footer and try allocate again
                 let footer = RegionFooter {
                     magic: REGION_MAGIC,
@@ -480,10 +480,11 @@ where
                 footer.write(slice.as_mut());
                 slice.destroy().await;
             }
-            crate::region::AllocateResult::Ok(slice) => {
+            AllocateResult::Ok(slice) => {
                 // region is empty, skip
                 slice.destroy().await
             }
+            AllocateResult::None => unreachable!(),
         }
         Ok(())
     }
@@ -521,10 +522,15 @@ where
 
         tracing::info!("finish store recovery, {} region recovered", recovered);
 
+        // Force trigger reclamation.
+        if recovered == self.device.regions() {
+            self.region_manager.clean_regions().flash();
+        }
+
         Ok(())
     }
 
-    /// return `true` if region is valid, otherwise `false`
+    /// Return `true` if region is valid, otherwise `false`
     async fn recover_region(
         region_id: RegionId,
         region_manager: Arc<RegionManager<D, EP, EL>>,
@@ -582,17 +588,23 @@ where
 
         self.metrics.bytes_insert.inc_by(serialized_len as u64);
 
-        let mut slice = match self.region_manager.allocate(serialized_len).await {
-            crate::region::AllocateResult::Ok(slice) => slice,
-            crate::region::AllocateResult::Full { mut slice, remain } => {
-                // current region is full, write region footer and try allocate again
-                let footer = RegionFooter {
-                    magic: REGION_MAGIC,
-                    padding: remain as u64,
-                };
-                footer.write(slice.as_mut());
-                slice.destroy().await;
-                self.region_manager.allocate(serialized_len).await.unwrap()
+        let mut slice = loop {
+            match self
+                .region_manager
+                .allocate(serialized_len, !writer.is_skippable)
+                .await
+            {
+                AllocateResult::Ok(slice) => break slice,
+                AllocateResult::Full { mut slice, remain } => {
+                    // current region is full, write region footer and try allocate again
+                    let footer = RegionFooter {
+                        magic: REGION_MAGIC,
+                        padding: remain as u64,
+                    };
+                    footer.write(slice.as_mut());
+                    slice.destroy().await;
+                }
+                AllocateResult::None => return Ok(false),
             }
         };
 
@@ -645,6 +657,7 @@ where
     duration: Duration,
 
     is_inserted: bool,
+    is_skippable: bool,
 }
 
 impl<'a, K, V, D, EP, EL> StoreWriter<'a, K, V, D, EP, EL>
@@ -664,6 +677,7 @@ where
             is_judged: false,
             duration: Duration::from_nanos(0),
             is_inserted: false,
+            is_skippable: false,
         }
     }
 
@@ -680,6 +694,10 @@ where
     pub async fn finish(self, value: V) -> Result<bool> {
         self.store.apply_writer(self, value).await
     }
+
+    pub fn set_skippable(&mut self) {
+        self.is_skippable = true
+    }
 }
 
 impl<'a, K, V, D, EP, EL> Debug for StoreWriter<'a, K, V, D, EP, EL>

From 5109637f51e9674057f35833b4d7e0d428fadd5d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 29 Jul 2023 21:50:44 +0800
Subject: [PATCH 070/261] refactor: modify shutdown runners order, add hakari
 check (#93)

* refactor: modify shutdown runners order, add hakari check

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* rename close

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix CI

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* add license header

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update readmer

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .config/hakari.toml                 | 24 ++++++++++
 .github/template/template.yml       |  8 +++-
 .github/workflows/main.yml          |  8 +++-
 .github/workflows/pull-request.yml  |  8 +++-
 Cargo.toml                          |  4 ++
 Makefile                            | 11 ++++-
 README.md                           |  2 +
 foyer-common/Cargo.toml             |  1 +
 foyer-intrusive/Cargo.toml          |  1 +
 foyer-memory/Cargo.toml             |  1 +
 foyer-storage-bench/Cargo.toml      |  1 +
 foyer-storage-bench/src/analyze.rs  |  6 +--
 foyer-storage-bench/src/main.rs     | 71 +++++++++++----------------
 foyer-storage/Cargo.toml            |  1 +
 foyer-storage/src/store.rs          | 74 ++++++++++++++++++-----------
 foyer-workspace-hack/.gitattributes |  4 ++
 foyer-workspace-hack/Cargo.toml     | 39 +++++++++++++++
 foyer-workspace-hack/build.rs       | 16 +++++++
 foyer-workspace-hack/src/lib.rs     | 15 ++++++
 foyer/Cargo.toml                    |  1 +
 20 files changed, 215 insertions(+), 81 deletions(-)
 create mode 100644 .config/hakari.toml
 create mode 100644 foyer-workspace-hack/.gitattributes
 create mode 100644 foyer-workspace-hack/Cargo.toml
 create mode 100644 foyer-workspace-hack/build.rs
 create mode 100644 foyer-workspace-hack/src/lib.rs

diff --git a/.config/hakari.toml b/.config/hakari.toml
new file mode 100644
index 00000000..3c115dc3
--- /dev/null
+++ b/.config/hakari.toml
@@ -0,0 +1,24 @@
+# This file contains settings for `cargo hakari`.
+# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options.
+
+hakari-package = "foyer-workspace-hack"
+
+# Format version for hakari's output. Version 4 requires cargo-hakari 0.9.22 or above.
+dep-format-version = "4"
+
+# Setting workspace.resolver = "2" in the root Cargo.toml is HIGHLY recommended.
+# Hakari works much better with the new feature resolver.
+# For more about the new feature resolver, see:
+# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver
+resolver = "2"
+
+# Add triples corresponding to platforms commonly used by developers here.
+# https://doc.rust-lang.org/rustc/platform-support.html
+platforms = [
+    # "x86_64-unknown-linux-gnu",
+    # "x86_64-apple-darwin",
+    # "x86_64-pc-windows-msvc",
+]
+
+# Write out exact versions rather than a semver range. (Defaults to false.)
+# exact-versions = true
diff --git a/.github/template/template.yml b/.github/template/template.yml
index 6ced2d33..100dbf80 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -47,10 +47,10 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
-      - name: Install cargo-sort
+      - name: Install cargo tools
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          cargo install cargo-sort
+          cargo install cargo-sort cargo-hakari
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
@@ -62,6 +62,10 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
+      - name: Run hakari check
+        run: |
+          cargo hakari generate --diff
+          cargo hakari manage-deps --dry-run
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5c1f8f9a..393b080a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -54,10 +54,10 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
-      - name: Install cargo-sort
+      - name: Install cargo tools
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          cargo install cargo-sort
+          cargo install cargo-sort cargo-hakari
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
@@ -69,6 +69,10 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
+      - name: Run hakari check
+        run: |
+          cargo hakari generate --diff
+          cargo hakari manage-deps --dry-run
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index fd5fd2c6..a01681d8 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -53,10 +53,10 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
-      - name: Install cargo-sort
+      - name: Install cargo tools
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          cargo install cargo-sort
+          cargo install cargo-sort cargo-hakari
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
@@ -68,6 +68,10 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
+      - name: Run hakari check
+        run: |
+          cargo hakari generate --diff
+          cargo hakari manage-deps --dry-run
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
diff --git a/Cargo.toml b/Cargo.toml
index d6f6620a..ec34e159 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,7 +7,11 @@ members = [
     "foyer-memory",
     "foyer-storage",
     "foyer-storage-bench",
+    "foyer-workspace-hack",
 ]
 
 [patch.crates-io]
 # cmsketch = { path = "../cmsketch-rs" }
+
+[profile.release]
+debug = 1
diff --git a/Makefile b/Makefile
index 4b05f67c..c45969cc 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,15 @@
 SHELL := /bin/bash
-.PHONY: proto check test
+.PHONY: proto check test deps
+
+deps:
+	cargo install cargo-hakari cargo-sort
 
 check:
-	cargo sort -w && cargo fmt --all && cargo clippy --all-targets
+	cargo hakari generate
+	cargo hakari manage-deps
+	cargo sort -w
+	cargo fmt --all
+	cargo clippy --all-targets
 
 test:
 	cargo test --all
\ No newline at end of file
diff --git a/README.md b/README.md
index 93e2dd59..0b616093 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
 # foyer
 
+[![CI (main)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml) [![License Checker](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml)
+
 *foyer* aims to be a user-friendly hybrid cache lib in Rust. 
 
 *foyer* is inspired by [Facebook/CacheLib](https://github.com/facebook/cachelib), which is an excellent hybrid cache lib in C++. *foyer* is not only a 'rewrite in Rust project', but provide some features that *CacheLib* doesn't have for now.
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 1019541d..cb8eb94c 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -9,6 +9,7 @@ license = "Apache-2.0"
 
 [dependencies]
 bytes = "1"
+foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 parking_lot = "0.12"
 paste = "1.0"
 rand = "0.8.5"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index c13664a5..2667713f 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -12,6 +12,7 @@ license = "Apache-2.0"
 bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
+foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 itertools = "0.10.5"
 memoffset = "0.8"
 parking_lot = "0.12"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 5a83d42f..ce5bfd32 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -15,6 +15,7 @@ bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
+foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 9145a090..0a3158ff 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -13,6 +13,7 @@ clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.1", optional = true }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-storage = { path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
 itertools = "0.10.5"
diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index 82a852cd..34ab4b1a 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -38,7 +38,7 @@ use std::{
 use bytesize::ByteSize;
 use hdrhistogram::Histogram;
 use parking_lot::RwLock;
-use tokio::sync::oneshot;
+use tokio::sync::broadcast;
 
 use crate::utils::{iostat, IoStat};
 
@@ -249,14 +249,14 @@ pub async fn monitor(
     iostat_path: impl AsRef<Path>,
     interval: Duration,
     metrics: Metrics,
-    mut stop: oneshot::Receiver<()>,
+    mut stop: broadcast::Receiver<()>,
 ) {
     let mut stat = iostat(&iostat_path);
     let mut metrics_dump = metrics.dump();
     loop {
         let start = Instant::now();
         match stop.try_recv() {
-            Err(oneshot::error::TryRecvError::Empty) => {}
+            Err(broadcast::error::TryRecvError::Empty) => {}
             _ => return,
         }
 
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index dfa31699..8f5a1ac9 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -44,7 +44,7 @@ use itertools::Itertools;
 use rand::{rngs::StdRng, Rng, SeedableRng};
 
 use rate::RateLimiter;
-use tokio::sync::oneshot;
+use tokio::sync::broadcast;
 use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
 
 #[derive(Parser, Debug, Clone)]
@@ -273,8 +273,7 @@ async fn main() {
 
     let store = TStore::open(config).await.unwrap();
 
-    let (iostat_stop_tx, iostat_stop_rx) = oneshot::channel();
-    let (bench_stop_tx, bench_stop_rx) = oneshot::channel();
+    let (stop_tx, _) = broadcast::channel(4096);
 
     let handle_monitor = tokio::spawn({
         let iostat_path = iostat_path.clone();
@@ -284,24 +283,24 @@ async fn main() {
             iostat_path,
             Duration::from_secs(args.report_interval),
             metrics,
-            iostat_stop_rx,
+            stop_tx.subscribe(),
         )
     });
-    let handle_signal = tokio::spawn(async move {
-        tokio::signal::ctrl_c().await.unwrap();
-        bench_stop_tx.send(()).unwrap();
-        iostat_stop_tx.send(()).unwrap();
-    });
+
     let handle_bench = tokio::spawn(bench(
         args.clone(),
         store.clone(),
         metrics.clone(),
-        bench_stop_rx,
+        stop_tx.clone(),
     ));
 
+    let handle_signal = tokio::spawn(async move {
+        tokio::signal::ctrl_c().await.unwrap();
+        tracing::warn!("foyer-storage-bench is cancelled with CTRL-C");
+        stop_tx.send(()).unwrap();
+    });
+
     handle_bench.await.unwrap();
-    handle_monitor.abort();
-    handle_signal.abort();
 
     let iostat_end = iostat(&iostat_path);
     let metrics_dump_end = metrics.dump();
@@ -312,12 +311,16 @@ async fn main() {
         &metrics_dump_start,
         &metrics_dump_end,
     );
-    println!("\nTotal:\n{}", analysis);
 
-    store.shutdown_runners().await.unwrap();
+    store.close().await.unwrap();
+
+    handle_monitor.abort();
+    handle_signal.abort();
+
+    println!("\nTotal:\n{}", analysis);
 }
 
-async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::Receiver<()>) {
+async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop_tx: broadcast::Sender<()>) {
     let w_rate = if args.w_rate == 0.0 {
         None
     } else {
@@ -331,14 +334,8 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::
 
     let index = Arc::new(AtomicU64::new(0));
 
-    let (w_stop_txs, w_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
-        (0..args.writers).map(|_| oneshot::channel()).unzip();
-    let (r_stop_txs, r_stop_rxs): (Vec<oneshot::Sender<()>>, Vec<oneshot::Receiver<()>>) =
-        (0..args.readers).map(|_| oneshot::channel()).unzip();
-
-    let w_handles = w_stop_rxs
-        .into_iter()
-        .map(|w_stop_rx| {
+    let w_handles = (0..args.writers)
+        .map(|_| {
             tokio::spawn(write(
                 args.entry_size,
                 w_rate,
@@ -346,13 +343,12 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::
                 store.clone(),
                 args.time,
                 metrics.clone(),
-                w_stop_rx,
+                stop_tx.subscribe(),
             ))
         })
         .collect_vec();
-    let r_handles = r_stop_rxs
-        .into_iter()
-        .map(|r_stop_rx| {
+    let r_handles = (0..args.readers)
+        .map(|_| {
             tokio::spawn(read(
                 args.entry_size,
                 r_rate,
@@ -360,23 +356,12 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop: oneshot::
                 store.clone(),
                 args.time,
                 metrics.clone(),
-                r_stop_rx,
+                stop_tx.subscribe(),
                 args.lookup_range,
             ))
         })
         .collect_vec();
 
-    tokio::spawn(async move {
-        if let Ok(()) = stop.await {
-            for w_stop_tx in w_stop_txs {
-                let _ = w_stop_tx.send(());
-            }
-            for r_stop_tx in r_stop_txs {
-                let _ = r_stop_tx.send(());
-            }
-        }
-    });
-
     join_all(w_handles).await;
     join_all(r_handles).await;
 }
@@ -389,7 +374,7 @@ async fn write(
     store: Arc<TStore>,
     time: u64,
     metrics: Metrics,
-    mut stop: oneshot::Receiver<()>,
+    mut stop: broadcast::Receiver<()>,
 ) {
     let start = Instant::now();
 
@@ -397,7 +382,7 @@ async fn write(
 
     loop {
         match stop.try_recv() {
-            Err(oneshot::error::TryRecvError::Empty) => {}
+            Err(broadcast::error::TryRecvError::Empty) => {}
             _ => return,
         }
         if start.elapsed().as_secs() >= time {
@@ -433,7 +418,7 @@ async fn read(
     store: Arc<TStore>,
     time: u64,
     metrics: Metrics,
-    mut stop: oneshot::Receiver<()>,
+    mut stop: broadcast::Receiver<()>,
     look_up_range: u64,
 ) {
     let start = Instant::now();
@@ -444,7 +429,7 @@ async fn read(
 
     loop {
         match stop.try_recv() {
-            Err(oneshot::error::TryRecvError::Empty) => {}
+            Err(broadcast::error::TryRecvError::Empty) => {}
             _ => return,
         }
         if start.elapsed().as_secs() >= time {
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index baa13504..1e02dd60 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -17,6 +17,7 @@ bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
+foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 0bf6de19..4810e846 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -47,6 +47,7 @@ use foyer_intrusive::core::adapter::Link;
 use std::hash::Hasher;
 
 const REGION_MAGIC: u64 = 0x19970327;
+const DEFAULT_BROADCAST_CAPACITY: usize = 4096;
 
 pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
 
@@ -145,9 +146,11 @@ where
 
     event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
 
-    handles: Mutex<Vec<JoinHandle<()>>>,
+    flusher_handles: Mutex<Vec<JoinHandle<()>>>,
+    flushers_stop_tx: broadcast::Sender<()>,
 
-    stop_tx: broadcast::Sender<()>,
+    reclaimer_handles: Mutex<Vec<JoinHandle<()>>>,
+    reclaimers_stop_tx: broadcast::Sender<()>,
 
     metrics: Arc<Metrics>,
 
@@ -178,12 +181,14 @@ where
 
         let indices = Arc::new(Indices::new(device.regions()));
 
-        let (stop_tx, _stop_rx) = broadcast::channel(config.flushers + config.reclaimers + 1);
+        let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
+        let (reclaimers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
+
         let flusher_stop_rxs = (0..config.flushers)
-            .map(|_| stop_tx.subscribe())
+            .map(|_| flushers_stop_tx.subscribe())
             .collect_vec();
         let reclaimer_stop_rxs = (0..config.reclaimers)
-            .map(|_| stop_tx.subscribe())
+            .map(|_| reclaimers_stop_tx.subscribe())
             .collect_vec();
 
         let metrics = match (
@@ -206,8 +211,10 @@ where
             admissions: config.admissions,
             reinsertions: config.reinsertions,
             event_listeners: config.event_listeners.clone(),
-            handles: Mutex::new(vec![]),
-            stop_tx,
+            flusher_handles: Mutex::new(vec![]),
+            reclaimer_handles: Mutex::new(vec![]),
+            flushers_stop_tx,
+            reclaimers_stop_tx,
             metrics: metrics.clone(),
             _marker: PhantomData,
         });
@@ -256,31 +263,44 @@ where
 
         store.recover(config.recover_concurrency).await?;
 
-        let mut handles = vec![];
-        handles.append(
-            &mut flushers
-                .into_iter()
-                .map(|flusher| tokio::spawn(async move { flusher.run().await.unwrap() }))
-                .collect_vec(),
-        );
-        handles.append(
-            &mut reclaimers
-                .into_iter()
-                .map(|reclaimer| tokio::spawn(async move { reclaimer.run().await.unwrap() }))
-                .collect_vec(),
-        );
-        store.handles.lock().append(&mut handles);
+        let flusher_handles = flushers
+            .into_iter()
+            .map(|flusher| tokio::spawn(async move { flusher.run().await.unwrap() }))
+            .collect_vec();
+
+        let reclaimer_handles = reclaimers
+            .into_iter()
+            .map(|reclaimer| tokio::spawn(async move { reclaimer.run().await.unwrap() }))
+            .collect_vec();
+
+        *store.flusher_handles.lock() = flusher_handles;
+        *store.reclaimer_handles.lock() = reclaimer_handles;
 
         Ok(store)
     }
 
-    pub async fn shutdown_runners(&self) -> Result<()> {
+    pub async fn close(&self) -> Result<()> {
+        // seal current dirty buffer and trigger flushing
         self.seal().await?;
-        self.stop_tx.send(()).unwrap();
-        let handles = self.handles.lock().drain(..).collect_vec();
+
+        // stop and wait for reclaimers
+        let handles = self.reclaimer_handles.lock().drain(..).collect_vec();
+        if !handles.is_empty() {
+            self.reclaimers_stop_tx.send(()).unwrap();
+        }
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        // stop and wait for flushers
+        let handles = self.flusher_handles.lock().drain(..).collect_vec();
+        if !handles.is_empty() {
+            self.flushers_stop_tx.send(()).unwrap();
+        }
         for handle in handles {
             handle.await.unwrap();
         }
+
         Ok(())
     }
 
@@ -1186,7 +1206,7 @@ pub mod tests {
             store.insert(i, vec![i as u8; 1 * MB]).await.unwrap();
         }
 
-        store.shutdown_runners().await.unwrap();
+        store.close().await.unwrap();
 
         let remains = recorder.remains();
 
@@ -1228,8 +1248,6 @@ pub mod tests {
         };
         let store = TestStore::open(config).await.unwrap();
 
-        store.shutdown_runners().await.unwrap();
-
         for i in 0..12 {
             if remains.contains(&i) {
                 assert_eq!(
@@ -1241,6 +1259,8 @@ pub mod tests {
             }
         }
 
+        store.close().await.unwrap();
+
         drop(store);
     }
 }
diff --git a/foyer-workspace-hack/.gitattributes b/foyer-workspace-hack/.gitattributes
new file mode 100644
index 00000000..3e9dba4b
--- /dev/null
+++ b/foyer-workspace-hack/.gitattributes
@@ -0,0 +1,4 @@
+# Avoid putting conflict markers in the generated Cargo.toml file, since their presence breaks
+# Cargo.
+# Also do not check out the file as CRLF on Windows, as that's what hakari needs.
+Cargo.toml merge=binary -crlf
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
new file mode 100644
index 00000000..61591aa3
--- /dev/null
+++ b/foyer-workspace-hack/Cargo.toml
@@ -0,0 +1,39 @@
+# This file is generated by `cargo hakari`.
+# To regenerate, run:
+#     cargo hakari generate
+
+[package]
+name = "foyer-workspace-hack"
+version = "0.1.0"
+description = "workspace-hack package, managed by hakari"
+# You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.
+publish = false
+# The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments
+# are managed by hakari.
+
+### BEGIN HAKARI SECTION
+[dependencies]
+crossbeam-channel = { version = "0.5" }
+crossbeam-utils = { version = "0.8" }
+either = { version = "1", default-features = false, features = ["use_std"] }
+futures-channel = { version = "0.3", features = ["sink"] }
+futures-core = { version = "0.3" }
+futures-sink = { version = "0.3" }
+itertools = { version = "0.10" }
+libc = { version = "0.2", features = ["extra_traits"] }
+memchr = { version = "2" }
+miniz_oxide = { version = "0.7", default-features = false, features = ["with-alloc"] }
+parking_lot = { version = "0.12", features = ["deadlock_detection"] }
+parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
+rand = { version = "0.8", features = ["small_rng"] }
+tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
+tracing-core = { version = "0.1" }
+
+[build-dependencies]
+either = { version = "1", default-features = false, features = ["use_std"] }
+itertools = { version = "0.10" }
+proc-macro2 = { version = "1" }
+quote = { version = "1" }
+syn = { version = "2", features = ["extra-traits", "full", "visit-mut"] }
+
+### END HAKARI SECTION
diff --git a/foyer-workspace-hack/build.rs b/foyer-workspace-hack/build.rs
new file mode 100644
index 00000000..b7889cc7
--- /dev/null
+++ b/foyer-workspace-hack/build.rs
@@ -0,0 +1,16 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// A build script is required for cargo to consider build dependencies.
+fn main() {}
diff --git a/foyer-workspace-hack/src/lib.rs b/foyer-workspace-hack/src/lib.rs
new file mode 100644
index 00000000..61f71adc
--- /dev/null
+++ b/foyer-workspace-hack/src/lib.rs
@@ -0,0 +1,15 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// This is a stub lib.rs.
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 07a22aea..1e937b6d 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -15,6 +15,7 @@ crossbeam = "0.8"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-storage = { path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"

From d3b2972484f024f94b0308594a4cb4bf55d83cd7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 1 Aug 2023 18:20:05 +0800
Subject: [PATCH 071/261] fix: ignore trivial error (#96)

ignored:
1. fetch value error
2. weight not equal error

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                          |  2 +-
 foyer-storage/src/device/error.rs | 50 +++++++++++++++++++++---
 foyer-storage/src/device/fs.rs    | 18 ++++-----
 foyer-storage/src/device/mod.rs   | 28 ++++++--------
 foyer-storage/src/error.rs        | 63 +++++++++++++++++++------------
 foyer-storage/src/lib.rs          |  2 +
 foyer-storage/src/store.rs        | 44 ++++++++++++++-------
 7 files changed, 137 insertions(+), 70 deletions(-)

diff --git a/Makefile b/Makefile
index c45969cc..f1ca3065 100644
--- a/Makefile
+++ b/Makefile
@@ -12,4 +12,4 @@ check:
 	cargo clippy --all-targets
 
 test:
-	cargo test --all
\ No newline at end of file
+	RUST_BACKTRACE=1 cargo test --all
\ No newline at end of file
diff --git a/foyer-storage/src/device/error.rs b/foyer-storage/src/device/error.rs
index 17ffc996..bdbbc004 100644
--- a/foyer-storage/src/device/error.rs
+++ b/foyer-storage/src/device/error.rs
@@ -12,20 +12,58 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::backtrace::Backtrace;
+
+#[derive(thiserror::Error, Debug)]
+#[error("{0}")]
+pub struct DeviceError(Box<DeviceErrorInner>);
+
+#[derive(thiserror::Error, Debug)]
+#[error("{source}")]
+struct DeviceErrorInner {
+    source: DeviceErrorKind,
+    backtrace: Backtrace,
+}
+
 #[derive(thiserror::Error, Debug)]
-pub enum Error {
+pub enum DeviceErrorKind {
     #[error("io error: {0}")]
     Io(#[from] std::io::Error),
     #[error("nix error: {0}")]
     Nix(#[from] nix::errno::Errno),
     #[error("other error: {0}")]
-    Other(String),
+    Other(#[from] Box<dyn std::error::Error + Send + Sync + 'static>),
+}
+
+impl From<std::io::Error> for DeviceError {
+    fn from(value: std::io::Error) -> Self {
+        value.into()
+    }
+}
+
+impl From<nix::errno::Errno> for DeviceError {
+    fn from(value: nix::errno::Errno) -> Self {
+        value.into()
+    }
 }
 
-impl Error {
-    pub fn io(e: std::io::Error) -> Self {
-        Self::Io(e)
+impl From<String> for DeviceError {
+    fn from(value: String) -> Self {
+        value.into()
     }
 }
 
-pub type Result<T> = core::result::Result<T, Error>;
+pub type DeviceResult<T> = std::result::Result<T, DeviceError>;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_error_size() {
+        assert_eq!(
+            std::mem::size_of::<DeviceError>(),
+            std::mem::size_of::<usize>()
+        );
+    }
+}
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 08968233..49333059 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -24,7 +24,7 @@ use crate::region::RegionId;
 use super::{
     allocator::AlignedAllocator,
     asyncify,
-    error::{Error, Result},
+    error::{DeviceError, DeviceResult},
     Device, IoBuf, IoBufMut,
 };
 use async_trait::async_trait;
@@ -79,7 +79,7 @@ impl Device for FsDevice {
     type Config = FsDeviceConfig;
     type IoBufferAllocator = AlignedAllocator;
 
-    async fn open(config: FsDeviceConfig) -> Result<Self> {
+    async fn open(config: FsDeviceConfig) -> DeviceResult<Self> {
         Self::open(config).await
     }
 
@@ -90,7 +90,7 @@ impl Device for FsDevice {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> Result<usize> {
+    ) -> DeviceResult<usize> {
         assert!(offset as usize + len <= self.inner.config.file_capacity);
 
         let fd = self.fd(region);
@@ -111,7 +111,7 @@ impl Device for FsDevice {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> Result<usize> {
+    ) -> DeviceResult<usize> {
         assert!(offset as usize + len <= self.inner.config.file_capacity);
 
         let fd = self.fd(region);
@@ -126,7 +126,7 @@ impl Device for FsDevice {
     }
 
     #[cfg(target_os = "linux")]
-    async fn flush(&self) -> Result<()> {
+    async fn flush(&self) -> DeviceResult<()> {
         let fd = self.inner.dir.as_raw_fd();
         // Commit fs cache to disk. Linux waits for I/O completions.
         //
@@ -143,7 +143,7 @@ impl Device for FsDevice {
     }
 
     #[cfg(not(target_os = "linux"))]
-    async fn flush(&self) -> Result<()> {
+    async fn flush(&self) -> DeviceResult<()> {
         // TODO(MrCroxx): track dirty files and call fsync(2) on them on other target os.
 
         Ok(())
@@ -178,7 +178,7 @@ impl Device for FsDevice {
 }
 
 impl FsDevice {
-    pub async fn open(config: FsDeviceConfig) -> Result<Self> {
+    pub async fn open(config: FsDeviceConfig) -> DeviceResult<Self> {
         config.verify();
 
         // TODO(MrCroxx): write and read config to a manifest file for pinning
@@ -207,9 +207,9 @@ impl FsDevice {
                     #[cfg(target_os = "linux")]
                     opts.custom_flags(libc::O_DIRECT);
 
-                    let file = opts.open(path).map_err(Error::io)?;
+                    let file = opts.open(path)?;
 
-                    Ok::<File, Error>(file)
+                    Ok::<_, DeviceError>(file)
                 }
             })
             .collect_vec();
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 044570cc..85a10a16 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -17,11 +17,10 @@ pub mod error;
 pub mod fs;
 
 use async_trait::async_trait;
-use error::Result;
-
 use std::{alloc::Allocator, fmt::Debug};
 
 use crate::region::RegionId;
+use error::DeviceResult;
 
 pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
 pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
@@ -32,7 +31,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     type IoBufferAllocator: BufferAllocator;
     type Config: Debug;
 
-    async fn open(config: Self::Config) -> Result<Self>;
+    async fn open(config: Self::Config) -> DeviceResult<Self>;
 
     async fn write(
         &self,
@@ -40,7 +39,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> Result<usize>;
+    ) -> DeviceResult<usize>;
 
     async fn read(
         &self,
@@ -48,9 +47,9 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> Result<usize>;
+    ) -> DeviceResult<usize>;
 
-    async fn flush(&self) -> Result<()>;
+    async fn flush(&self) -> DeviceResult<()>;
 
     fn capacity(&self) -> usize;
 
@@ -71,17 +70,14 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
 }
 
 #[tracing::instrument(level = "trace", skip(f))]
-async fn asyncify<F, T>(f: F) -> error::Result<T>
+async fn asyncify<F, T>(f: F) -> DeviceResult<T>
 where
-    F: FnOnce() -> error::Result<T> + Send + 'static,
+    F: FnOnce() -> DeviceResult<T> + Send + 'static,
     T: Send + 'static,
 {
     match tokio::task::spawn_blocking(f).await {
         Ok(res) => res,
-        Err(e) => Err(error::Error::Other(format!(
-            "background task failed: {:?}",
-            e,
-        ))),
+        Err(e) => Err(format!("background task failed: {:?}", e,).into()),
     }
 }
 
@@ -103,7 +99,7 @@ pub mod tests {
         type Config = usize;
         type IoBufferAllocator = AlignedAllocator;
 
-        async fn open(config: usize) -> Result<Self> {
+        async fn open(config: usize) -> DeviceResult<Self> {
             Ok(Self::new(config))
         }
 
@@ -113,7 +109,7 @@ pub mod tests {
             _region: RegionId,
             _offset: u64,
             _len: usize,
-        ) -> Result<usize> {
+        ) -> DeviceResult<usize> {
             Ok(0)
         }
 
@@ -123,11 +119,11 @@ pub mod tests {
             _region: RegionId,
             _offset: u64,
             _len: usize,
-        ) -> Result<usize> {
+        ) -> DeviceResult<usize> {
             Ok(0)
         }
 
-        async fn flush(&self) -> Result<()> {
+        async fn flush(&self) -> DeviceResult<()> {
             Ok(())
         }
 
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index 24996b5d..d4dde62a 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -12,42 +12,55 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::backtrace::Backtrace;
+
+use crate::device::error::DeviceError;
+
+#[derive(thiserror::Error, Debug)]
+#[error("{0}")]
+pub struct Error(Box<ErrorInner>);
+
 #[derive(thiserror::Error, Debug)]
-pub enum Error {
+#[error("{source}")]
+struct ErrorInner {
+    source: ErrorKind,
+    backtrace: Backtrace,
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum ErrorKind {
     #[error("device error: {0}")]
-    Device(#[from] super::device::error::Error),
-    #[error("entry too large: {len} > {capacity}")]
-    EntryTooLarge { len: usize, capacity: usize },
-    #[error("checksum mismatch, checksum: {checksum}, expected: {expected}")]
-    ChecksumMismatch { checksum: u64, expected: u64 },
-    #[error("channel full")]
-    ChannelFull,
-    #[error("event listener error: {0}")]
-    EventListener(Box<dyn std::error::Error + Send + Sync + 'static>),
-    #[error("fetch value error: {0}")]
-    FetchValue(anyhow::Error),
+    Device(#[from] DeviceError),
     #[error("other error: {0}")]
     Other(#[from] anyhow::Error),
 }
 
-impl Error {
-    pub fn device(e: super::device::error::Error) -> Self {
-        Self::Device(e)
-    }
-
-    pub fn fetch_value(e: impl Into<anyhow::Error>) -> Self {
-        Self::Other(e.into())
+impl From<ErrorKind> for Error {
+    fn from(value: ErrorKind) -> Self {
+        value.into()
     }
+}
 
-    pub fn other(e: impl Into<anyhow::Error>) -> Self {
-        Self::Other(e.into())
+impl From<DeviceError> for Error {
+    fn from(value: DeviceError) -> Self {
+        value.into()
     }
+}
 
-    pub fn event_listener(
-        e: impl Into<Box<dyn std::error::Error + Send + Sync + 'static>>,
-    ) -> Self {
-        Self::EventListener(e.into())
+impl From<anyhow::Error> for Error {
+    fn from(value: anyhow::Error) -> Self {
+        value.into()
     }
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_error_size() {
+        assert_eq!(std::mem::size_of::<Error>(), std::mem::size_of::<usize>());
+    }
+}
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index e1370c2e..1c4fd29c 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -17,6 +17,8 @@
 #![feature(trait_alias)]
 #![feature(get_mut_unchecked)]
 #![feature(let_chains)]
+#![feature(provide_any)]
+#![feature(error_generic_member_access)]
 #![allow(clippy::type_complexity)]
 
 pub mod admission;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 4810e846..b6bfcab0 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -22,7 +22,7 @@ use std::{
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, rate::RateLimiter};
 use foyer_intrusive::eviction::EvictionPolicy;
-use futures::Future;
+use futures::{future::try_join_all, Future};
 use itertools::Itertools;
 use parking_lot::Mutex;
 use tokio::{sync::broadcast, task::JoinHandle};
@@ -31,7 +31,7 @@ use twox_hash::XxHash64;
 use crate::{
     admission::AdmissionPolicy,
     device::Device,
-    error::{Error, Result},
+    error::Result,
     event::EventListener,
     flusher::Flusher,
     indices::{Index, Indices},
@@ -334,7 +334,13 @@ where
         if !writer.judge() {
             return Ok(false);
         }
-        let value = f().map_err(Error::fetch_value)?;
+        let value = match f() {
+            Ok(value) => value,
+            Err(e) => {
+                tracing::warn!("fetch value error: {:?}", e);
+                return Ok(false);
+            }
+        };
         writer.finish(value).await
     }
 
@@ -354,7 +360,13 @@ where
         if !writer.judge() {
             return Ok(false);
         }
-        let value = f().await.map_err(Error::fetch_value)?;
+        let value = match f().await {
+            Ok(value) => value,
+            Err(e) => {
+                tracing::warn!("fetch value error: {:?}", e);
+                return Ok(false);
+            }
+        };
         writer.finish(value).await
     }
 
@@ -533,10 +545,13 @@ where
         }
 
         let mut recovered = 0;
-        for (region_id, handle) in handles.into_iter().enumerate() {
-            if handle.await.map_err(Error::other)?? {
+
+        let results = try_join_all(handles).await.map_err(anyhow::Error::from)?;
+
+        for (region_id, result) in results.into_iter().enumerate() {
+            if result? {
                 tracing::debug!("region {} is recovered", region_id);
-                recovered += 1;
+                recovered += 1
             }
         }
 
@@ -604,7 +619,12 @@ where
         }
 
         let serialized_len = self.serialized_len(key, &value);
-        assert_eq!(key.serialized_len() + value.serialized_len(), writer.weight);
+        if key.serialized_len() + value.serialized_len() == writer.weight {
+            tracing::error!(
+                "weight != key.serialized_len() + value.serialized_len(), weight: {}, key size: {}, value size: {}, key: {:?}",
+                writer.weight, key.serialized_len(), value.serialized_len(), key
+            );
+        }
 
         self.metrics.bytes_insert.inc_by(serialized_len as u64);
 
@@ -903,11 +923,9 @@ where
     let checksum = checksum(&buf[..offset]);
     if checksum != footer.checksum {
         tracing::warn!(
-            "read entry error: {}",
-            Error::ChecksumMismatch {
-                checksum,
-                expected: footer.checksum,
-            }
+            "checksum mismatch, checksum: {}, expected: {}",
+            checksum,
+            footer.checksum
         );
         return None;
     }

From 94518d1d4d4945e8b2e2c844f8e6de2eaa8715af Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 1 Aug 2023 23:32:49 +0800
Subject: [PATCH 072/261] feat: add and use simple fifo by default (#97)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/eviction/fifo.rs  | 207 +++---------
 foyer-intrusive/src/eviction/mod.rs   |   1 +
 foyer-intrusive/src/eviction/sfifo.rs | 456 ++++++++++++++++++++++++++
 foyer-memory/src/lib.rs               |   9 +-
 foyer-storage/src/lib.rs              |  14 +-
 foyer-storage/src/region_manager.rs   |   5 +-
 foyer-storage/src/store.rs            |   8 +-
 7 files changed, 523 insertions(+), 177 deletions(-)
 create mode 100644 foyer-intrusive/src/eviction/sfifo.rs

diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index c7158a37..c788765e 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -28,12 +28,10 @@
 
 use std::{mem::ManuallyDrop, ptr::NonNull};
 
-use itertools::Itertools;
-
 use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
-        adapter::{Adapter, Link, PriorityAdapter},
+        adapter::{Adapter, Link},
         pointer::PointerOps,
     },
     intrusive_adapter,
@@ -42,20 +40,11 @@ use crate::{
 use super::EvictionPolicy;
 
 #[derive(Debug, Clone)]
-pub struct FifoConfig {
-    /// `segment_ratios` is used to compute the ratio of each segment's size.
-    ///
-    /// The formula is as follows:
-    ///
-    /// `segment's size = total_segments * (segment's ratio / sum(ratio))`
-    pub segment_ratios: Vec<usize>,
-}
+pub struct FifoConfig;
 
 #[derive(Debug, Default)]
 pub struct FifoLink {
-    link_queue: DListLink,
-
-    priority: usize,
+    link: DListLink,
 }
 
 impl FifoLink {
@@ -66,53 +55,21 @@ impl FifoLink {
 
 impl Link for FifoLink {
     fn is_linked(&self) -> bool {
-        self.link_queue.is_linked()
+        self.link.is_linked()
     }
 }
 
-intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link_queue: DListLink } }
-
-/// Segmented FIFO policy
-///
-/// It divides the fifo queue into N segments. Each segment holds
-/// number of items proportional to its segment ratio. For example,
-/// if we have 3 segments and the ratio of [2, 1, 1], the lowest
-/// priority segment will hold 50% of the items whereas the other
-/// two higher priority segments will hold 25% each.
-///
-/// On insertion, a priority is used as an Insertion Point. E.g. a pri-2
-/// region will be inserted into the third highest priority segment. After
-/// the insertion is completed, we will trigger rebalance, where this
-/// region may be moved to below the insertion point, if the segment it
-/// was originally inserted into had exceeded the size allowed by its ratio.
-///
-/// Our rebalancing scheme allows the lowest priority segment to grow beyond
-/// its ratio allows for, since there is no lower segment to move into.
-///
-/// Also note that rebalancing is also triggered after an eviction.
-///
-/// The rate of inserting new regions look like the following:
-/// Pri-2 ---
-///          \
-/// Pri-1 -------
-///              \
-/// Pri-0 ------------
-/// When pri-2 exceeds its ratio, it effectively downgrades the oldest region in
-/// pri-2 to pri-1, and that region is now pushed down at the combined rate of
-/// (new pri-1 regions + new pri-2 regions), so effectively it gets evicted out
-/// of the system faster once it is beyond the pri-2 segment ratio. Segment
-/// ratio is put in place to prevent the lower segments getting so small a
-/// portion of the flash device.
+intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DListLink } }
+
+/// FIFO policy
 pub struct Fifo<A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
-    // Note: All queue share the same dlist link.
-    segments: Vec<DList<FifoLinkAdapter>>,
+    queue: DList<FifoLinkAdapter>,
 
-    config: FifoConfig,
-    total_ratio: usize,
+    _config: FifoConfig,
 
     len: usize,
 
@@ -121,7 +78,7 @@ where
 
 impl<A> Drop for Fifo<A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     fn drop(&mut self) {
@@ -137,20 +94,14 @@ where
 
 impl<A> Fifo<A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     pub fn new(config: FifoConfig) -> Self {
-        let segments = (0..config.segment_ratios.len())
-            .map(|_| DList::new())
-            .collect_vec();
-        let total_ratio = config.segment_ratios.iter().sum();
-
         Self {
-            segments,
+            queue: DList::new(),
 
-            config,
-            total_ratio,
+            _config: config,
 
             len: 0,
 
@@ -161,16 +112,11 @@ where
     fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
         unsafe {
             let item = self.adapter.pointer_ops().into_raw(ptr);
-            let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
+            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
 
             assert!(!link.as_ref().is_linked());
 
-            let priority = *self.adapter.item2priority(item);
-            link.as_mut().priority = priority.into();
-
-            self.segments[priority.into()].push_back(link);
-
-            self.rebalance();
+            self.queue.push_back(link);
 
             self.len += 1;
         }
@@ -186,14 +132,11 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            let priority = link.as_ref().priority;
-            self.segments[priority]
-                .iter_mut_from_raw(link.as_ref().link_queue.raw())
+            self.queue
+                .iter_mut_from_raw(link.as_ref().link.raw())
                 .remove()
                 .unwrap();
 
-            self.rebalance();
-
             self.len -= 1;
 
             self.adapter.pointer_ops().from_raw(item)
@@ -206,38 +149,13 @@ where
         self.len
     }
 
-    fn rebalance(&mut self) {
-        unsafe {
-            let total: usize = self.segments.iter().map(|queue| queue.len()).sum();
-
-            // Rebalance from highest-pri segment to lowest-pri segment. This means the
-            // lowest-pri segment can grow to far larger than its ratio suggests. This
-            // is okay, as we only need higher-pri segments for items that are deemed
-            // important.
-            // e.g. {[a, b, c], [d], [e]} is a valid state for a SFIFO with 3 segments
-            //      and a segment ratio of [1, 1, 1]
-            for high in (1..self.segments.len()).rev() {
-                let low = high - 1;
-                let limit = (total as f64 * self.config.segment_ratios[high] as f64
-                    / self.total_ratio as f64) as usize;
-                while self.segments[high].len() > limit {
-                    let mut link = self.segments[high].pop_front().unwrap();
-                    link.as_mut().priority = low;
-                    self.segments[low].push_back(link);
-                }
-            }
-        }
-    }
-
     fn iter(&self) -> FifoIter<A> {
-        let mut iter_segments = self.segments.iter().map(|queue| queue.iter()).collect_vec();
-
-        iter_segments.iter_mut().for_each(|iter| iter.front());
+        let mut iter = self.queue.iter();
+        iter.front();
 
         FifoIter {
-            lfu: self,
-            iter_segments,
-            segment: 0,
+            fifo: self,
+            iter,
 
             ptr: ManuallyDrop::new(None),
         }
@@ -246,26 +164,25 @@ where
 
 pub struct FifoIter<'a, A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
-    lfu: &'a Fifo<A>,
-    iter_segments: Vec<DListIter<'a, FifoLinkAdapter>>,
-    segment: usize,
+    fifo: &'a Fifo<A>,
+    iter: DListIter<'a, FifoLinkAdapter>,
 
     ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
 }
 
 impl<'a, A> FifoIter<'a, A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     unsafe fn update_ptr(&mut self, link: NonNull<FifoLink>) {
         std::mem::forget(self.ptr.take());
 
-        let item = self.lfu.adapter.link2item(link.as_ptr());
-        let ptr = self.lfu.adapter.pointer_ops().from_raw(item);
+        let item = self.fifo.adapter.link2item(link.as_ptr());
+        let ptr = self.fifo.adapter.pointer_ops().from_raw(item);
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
@@ -281,31 +198,23 @@ where
 
 impl<'a, A> Iterator for FifoIter<'a, A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     type Item = &'a <A::PointerOps as PointerOps>::Pointer;
 
     fn next(&mut self) -> Option<Self::Item> {
         unsafe {
-            let mut link = None;
-            while self.segment < self.iter_segments.len() {
-                match self.iter_segments[self.segment].get().map(|l| l.raw()) {
-                    Some(l) => {
-                        self.iter_segments[self.segment].next();
-                        link = Some(l);
-                        break;
-                    }
-                    None => self.segment += 1,
-                }
-            }
-            match link {
-                None => None,
+            let link = match self.iter.get() {
                 Some(link) => {
-                    self.update_ptr(link);
-                    self.ptr()
+                    let link = link.raw();
+                    self.iter.next();
+                    link
                 }
-            }
+                None => return None,
+            };
+            self.update_ptr(link);
+            self.ptr()
         }
     }
 }
@@ -314,14 +223,14 @@ where
 
 unsafe impl<A> Send for Fifo<A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 
 unsafe impl<A> Sync for Fifo<A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
@@ -332,21 +241,21 @@ unsafe impl Sync for FifoLink {}
 
 unsafe impl<'a, A> Send for FifoIter<'a, A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 
 unsafe impl<'a, A> Sync for FifoIter<'a, A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
 }
 
 impl<A> EvictionPolicy for Fifo<A>
 where
-    A: Adapter<Link = FifoLink> + PriorityAdapter<Link = FifoLink>,
+    A: Adapter<Link = FifoLink>,
     <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
 {
     type Adapter = A;
@@ -388,62 +297,48 @@ mod tests {
     use crate::eviction::EvictionPolicyExt;
     use itertools::Itertools;
 
-    use crate::priority_adapter;
-
     use super::*;
 
     #[derive(Debug)]
     struct FifoItem {
         link: FifoLink,
         key: u64,
-        priority: usize,
     }
 
     impl FifoItem {
-        fn new(key: u64, priority: usize) -> Self {
+        fn new(key: u64) -> Self {
             Self {
                 link: FifoLink::default(),
                 key,
-                priority,
             }
         }
     }
 
     intrusive_adapter! { FifoItemAdapter = Arc<FifoItem>: FifoItem { link: FifoLink } }
-    priority_adapter! { FifoItemAdapter = FifoItem { priority: usize } }
 
     #[test]
     fn test_fifo_simple() {
-        let config = FifoConfig {
-            segment_ratios: vec![6, 3, 1],
-        };
+        let config = FifoConfig;
         let mut fifo = Fifo::<FifoItemAdapter>::new(config);
 
         let mut items = vec![];
 
-        // see comments in `rebalance`
-        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 8], [9]]
         for key in 0..10 {
-            let item = Arc::new(FifoItem::new(key, 2));
+            let item = Arc::new(FifoItem::new(key));
             fifo.push(item.clone());
             items.push(item);
         }
         let v = fifo.iter().map(|item| item.key).collect_vec();
         assert_eq!(v, (0..10).collect_vec());
-        let lens = fifo.segments.iter().map(|queue| queue.len()).collect_vec();
-        assert_eq!(lens, vec![7, 2, 1]);
 
-        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 8, 10], [9]]
-        let item = Arc::new(FifoItem::new(10, 1));
-        fifo.push(item.clone());
-        items.push(item);
-        let v = fifo.iter().map(|item| item.key).collect_vec();
-        assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 9]);
+        let v = (0..5)
+            .map(|_| fifo.pop().unwrap())
+            .map(|item| item.key)
+            .collect_vec();
+        assert_eq!(v, (0..5).collect_vec());
 
-        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 10], [9]]
-        fifo.remove(&items[8]);
         let v = fifo.iter().map(|item| item.key).collect_vec();
-        assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 10, 9]);
+        assert_eq!(v, (5..10).collect_vec());
 
         drop(fifo);
 
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index c44152d0..aeb966ba 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -77,3 +77,4 @@ where
 pub mod fifo;
 pub mod lfu;
 pub mod lru;
+pub mod sfifo;
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
new file mode 100644
index 00000000..2f5445d3
--- /dev/null
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -0,0 +1,456 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{mem::ManuallyDrop, ptr::NonNull};
+
+use itertools::Itertools;
+
+use crate::{
+    collections::dlist::{DList, DListIter, DListLink},
+    core::{
+        adapter::{Adapter, Link, PriorityAdapter},
+        pointer::PointerOps,
+    },
+    intrusive_adapter,
+};
+
+use super::EvictionPolicy;
+
+#[derive(Debug, Clone)]
+pub struct SegmentedFifoConfig {
+    /// `segment_ratios` is used to compute the ratio of each segment's size.
+    ///
+    /// The formula is as follows:
+    ///
+    /// `segment's size = total_segments * (segment's ratio / sum(ratio))`
+    pub segment_ratios: Vec<usize>,
+}
+
+#[derive(Debug, Default)]
+pub struct SegmentedFifoLink {
+    link_queue: DListLink,
+
+    priority: usize,
+}
+
+impl SegmentedFifoLink {
+    fn raw(&self) -> NonNull<Self> {
+        unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
+    }
+}
+
+impl Link for SegmentedFifoLink {
+    fn is_linked(&self) -> bool {
+        self.link_queue.is_linked()
+    }
+}
+
+intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: SegmentedFifoLink { link_queue: DListLink } }
+
+/// Segmented FIFO policy
+///
+/// It divides the fifo queue into N segments. Each segment holds
+/// number of items proportional to its segment ratio. For example,
+/// if we have 3 segments and the ratio of [2, 1, 1], the lowest
+/// priority segment will hold 50% of the items whereas the other
+/// two higher priority segments will hold 25% each.
+///
+/// On insertion, a priority is used as an Insertion Point. E.g. a pri-2
+/// region will be inserted into the third highest priority segment. After
+/// the insertion is completed, we will trigger rebalance, where this
+/// region may be moved to below the insertion point, if the segment it
+/// was originally inserted into had exceeded the size allowed by its ratio.
+///
+/// Our rebalancing scheme allows the lowest priority segment to grow beyond
+/// its ratio allows for, since there is no lower segment to move into.
+///
+/// Also note that rebalancing is also triggered after an eviction.
+///
+/// The rate of inserting new regions look like the following:
+/// Pri-2 ---
+///          \
+/// Pri-1 -------
+///              \
+/// Pri-0 ------------
+/// When pri-2 exceeds its ratio, it effectively downgrades the oldest region in
+/// pri-2 to pri-1, and that region is now pushed down at the combined rate of
+/// (new pri-1 regions + new pri-2 regions), so effectively it gets evicted out
+/// of the system faster once it is beyond the pri-2 segment ratio. Segment
+/// ratio is put in place to prevent the lower segments getting so small a
+/// portion of the flash device.
+pub struct SegmentedFifo<A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    // Note: All queue share the same dlist link.
+    segments: Vec<DList<SegmentedFifoLinkAdapter>>,
+
+    config: SegmentedFifoConfig,
+    total_ratio: usize,
+
+    len: usize,
+
+    adapter: A,
+}
+
+impl<A> Drop for SegmentedFifo<A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    fn drop(&mut self) {
+        let mut to_remove = vec![];
+        for ptr in self.iter() {
+            to_remove.push(ptr.clone());
+        }
+        for ptr in to_remove {
+            self.remove(&ptr);
+        }
+    }
+}
+
+impl<A> SegmentedFifo<A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    pub fn new(config: SegmentedFifoConfig) -> Self {
+        let segments = (0..config.segment_ratios.len())
+            .map(|_| DList::new())
+            .collect_vec();
+        let total_ratio = config.segment_ratios.iter().sum();
+
+        Self {
+            segments,
+
+            config,
+            total_ratio,
+
+            len: 0,
+
+            adapter: A::new(),
+        }
+    }
+
+    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+        unsafe {
+            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let mut link =
+                NonNull::new_unchecked(self.adapter.item2link(item) as *mut SegmentedFifoLink);
+
+            assert!(!link.as_ref().is_linked());
+
+            let priority = *self.adapter.item2priority(item);
+            link.as_mut().priority = priority.into();
+
+            self.segments[priority.into()].push_back(link);
+
+            self.rebalance();
+
+            self.len += 1;
+        }
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<A::PointerOps as PointerOps>::Pointer,
+    ) -> <A::PointerOps as PointerOps>::Pointer {
+        unsafe {
+            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let link =
+                NonNull::new_unchecked(self.adapter.item2link(item) as *mut SegmentedFifoLink);
+
+            assert!(link.as_ref().is_linked());
+
+            let priority = link.as_ref().priority;
+            self.segments[priority]
+                .iter_mut_from_raw(link.as_ref().link_queue.raw())
+                .remove()
+                .unwrap();
+
+            self.rebalance();
+
+            self.len -= 1;
+
+            self.adapter.pointer_ops().from_raw(item)
+        }
+    }
+
+    fn access(&mut self, _ptr: &<A::PointerOps as PointerOps>::Pointer) {}
+
+    fn len(&self) -> usize {
+        self.len
+    }
+
+    fn rebalance(&mut self) {
+        unsafe {
+            let total: usize = self.segments.iter().map(|queue| queue.len()).sum();
+
+            // Rebalance from highest-pri segment to lowest-pri segment. This means the
+            // lowest-pri segment can grow to far larger than its ratio suggests. This
+            // is okay, as we only need higher-pri segments for items that are deemed
+            // important.
+            // e.g. {[a, b, c], [d], [e]} is a valid state for a SFIFO with 3 segments
+            //      and a segment ratio of [1, 1, 1]
+            for high in (1..self.segments.len()).rev() {
+                let low = high - 1;
+                let limit = (total as f64 * self.config.segment_ratios[high] as f64
+                    / self.total_ratio as f64) as usize;
+                while self.segments[high].len() > limit {
+                    let mut link = self.segments[high].pop_front().unwrap();
+                    link.as_mut().priority = low;
+                    self.segments[low].push_back(link);
+                }
+            }
+        }
+    }
+
+    fn iter(&self) -> SegmentedFifoIter<A> {
+        let mut iter_segments = self.segments.iter().map(|queue| queue.iter()).collect_vec();
+
+        iter_segments.iter_mut().for_each(|iter| iter.front());
+
+        SegmentedFifoIter {
+            sfifo: self,
+            iter_segments,
+            segment: 0,
+
+            ptr: ManuallyDrop::new(None),
+        }
+    }
+}
+
+pub struct SegmentedFifoIter<'a, A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    sfifo: &'a SegmentedFifo<A>,
+    iter_segments: Vec<DListIter<'a, SegmentedFifoLinkAdapter>>,
+    segment: usize,
+
+    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+}
+
+impl<'a, A> SegmentedFifoIter<'a, A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    unsafe fn update_ptr(&mut self, link: NonNull<SegmentedFifoLink>) {
+        std::mem::forget(self.ptr.take());
+
+        let item = self.sfifo.adapter.link2item(link.as_ptr());
+        let ptr = self.sfifo.adapter.pointer_ops().from_raw(item);
+        self.ptr = ManuallyDrop::new(Some(ptr));
+    }
+
+    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+        if self.ptr.is_none() {
+            return None;
+        }
+        let ptr = self.ptr.as_ref().unwrap();
+        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        Some(&*raw)
+    }
+}
+
+impl<'a, A> Iterator for SegmentedFifoIter<'a, A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        unsafe {
+            let mut link = None;
+            while self.segment < self.iter_segments.len() {
+                match self.iter_segments[self.segment].get().map(|l| l.raw()) {
+                    Some(l) => {
+                        self.iter_segments[self.segment].next();
+                        link = Some(l);
+                        break;
+                    }
+                    None => self.segment += 1,
+                }
+            }
+            match link {
+                None => None,
+                Some(link) => {
+                    self.update_ptr(link);
+                    self.ptr()
+                }
+            }
+        }
+    }
+}
+
+// unsafe impl `Send + Sync` for structs with `NonNull` usage
+
+unsafe impl<A> Send for SegmentedFifo<A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+unsafe impl<A> Sync for SegmentedFifo<A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+unsafe impl Send for SegmentedFifoLink {}
+
+unsafe impl Sync for SegmentedFifoLink {}
+
+unsafe impl<'a, A> Send for SegmentedFifoIter<'a, A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+unsafe impl<'a, A> Sync for SegmentedFifoIter<'a, A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+}
+
+impl<A> EvictionPolicy for SegmentedFifo<A>
+where
+    A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
+    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+{
+    type Adapter = A;
+
+    type Config = SegmentedFifoConfig;
+
+    fn new(config: Self::Config) -> Self {
+        Self::new(config)
+    }
+
+    fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        self.insert(ptr)
+    }
+
+    fn remove(
+        &mut self,
+        ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
+    ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
+        self.remove(ptr)
+    }
+
+    fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+        self.access(ptr)
+    }
+
+    fn len(&self) -> usize {
+        self.len()
+    }
+
+    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
+        self.iter()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use crate::eviction::EvictionPolicyExt;
+    use itertools::Itertools;
+
+    use crate::priority_adapter;
+
+    use super::*;
+
+    #[derive(Debug)]
+    struct SegmentedFifoItem {
+        link: SegmentedFifoLink,
+        key: u64,
+        priority: usize,
+    }
+
+    impl SegmentedFifoItem {
+        fn new(key: u64, priority: usize) -> Self {
+            Self {
+                link: SegmentedFifoLink::default(),
+                key,
+                priority,
+            }
+        }
+    }
+
+    intrusive_adapter! { SegmentedFifoItemAdapter = Arc<SegmentedFifoItem>: SegmentedFifoItem { link: SegmentedFifoLink } }
+    priority_adapter! { SegmentedFifoItemAdapter = SegmentedFifoItem { priority: usize } }
+
+    #[test]
+    fn test_sfifo_simple() {
+        let config = SegmentedFifoConfig {
+            segment_ratios: vec![6, 3, 1],
+        };
+        let mut fifo = SegmentedFifo::<SegmentedFifoItemAdapter>::new(config);
+
+        let mut items = vec![];
+
+        // see comments in `rebalance`
+        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 8], [9]]
+        for key in 0..10 {
+            let item = Arc::new(SegmentedFifoItem::new(key, 2));
+            fifo.push(item.clone());
+            items.push(item);
+        }
+        let v = fifo.iter().map(|item| item.key).collect_vec();
+        assert_eq!(v, (0..10).collect_vec());
+        let lens = fifo.segments.iter().map(|queue| queue.len()).collect_vec();
+        assert_eq!(lens, vec![7, 2, 1]);
+
+        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 8, 10], [9]]
+        let item = Arc::new(SegmentedFifoItem::new(10, 1));
+        fifo.push(item.clone());
+        items.push(item);
+        let v = fifo.iter().map(|item| item.key).collect_vec();
+        assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 9]);
+
+        // inner: [[0, 1, 2, 3, 4, 5, 6], [7, 10], [9]]
+        fifo.remove(&items[8]);
+        let v = fifo.iter().map(|item| item.key).collect_vec();
+        assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 10, 9]);
+
+        drop(fifo);
+
+        for item in items {
+            assert_eq!(Arc::strong_count(&item), 1);
+        }
+    }
+}
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index fdaf832f..dd1bbaca 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -206,7 +206,7 @@ where
 
 #[cfg(test)]
 mod tests {
-    use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
+    use foyer_intrusive::eviction::sfifo::{SegmentedFifo, SegmentedFifoConfig, SegmentedFifoLink};
 
     use super::*;
 
@@ -228,8 +228,9 @@ mod tests {
         }
     }
 
-    type FifoCacheConfig = CacheConfig<Fifo<CacheItemEpAdapter<K, V, FifoLink>>>;
-    type FifoCache = Cache<K, V, Fifo<CacheItemEpAdapter<K, V, FifoLink>>, FifoLink>;
+    type FifoCacheConfig = CacheConfig<SegmentedFifo<CacheItemEpAdapter<K, V, SegmentedFifoLink>>>;
+    type FifoCache =
+        Cache<K, V, SegmentedFifo<CacheItemEpAdapter<K, V, SegmentedFifoLink>>, SegmentedFifoLink>;
 
     #[test]
     fn test_fifo_cache_simple() {
@@ -237,7 +238,7 @@ mod tests {
             capacity: 10,
             shard_bits: 0, // 1 shard
             hashmap_bits: 6,
-            eviction_config: FifoConfig {
+            eviction_config: SegmentedFifoConfig {
                 segment_ratios: vec![1],
             },
         };
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 1c4fd29c..b19927ce 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -74,21 +74,21 @@ pub type LfuFsStoreConfig<K, V> = store::StoreConfig<
     >,
 >;
 
-pub type FifoFsStore<K, V> = store::Store<
+pub type SegmentedFifoFsStore<K, V> = store::Store<
     K,
     V,
     device::fs::FsDevice,
-    foyer_intrusive::eviction::fifo::Fifo<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
+    foyer_intrusive::eviction::sfifo::SegmentedFifo<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::sfifo::SegmentedFifoLink>,
     >,
-    foyer_intrusive::eviction::fifo::FifoLink,
+    foyer_intrusive::eviction::sfifo::SegmentedFifoLink,
 >;
 
-pub type FifoFsStoreConfig<K, V> = store::StoreConfig<
+pub type SegmentedFifoFsStoreConfig<K, V> = store::StoreConfig<
     K,
     V,
     device::fs::FsDevice,
-    foyer_intrusive::eviction::fifo::Fifo<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
+    foyer_intrusive::eviction::sfifo::SegmentedFifo<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::sfifo::SegmentedFifoLink>,
     >,
 >;
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 10618817..37c9781c 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -21,7 +21,7 @@ use foyer_common::{
 use foyer_intrusive::{
     core::adapter::Link,
     eviction::{EvictionPolicy, EvictionPolicyExt},
-    intrusive_adapter, key_adapter, priority_adapter,
+    intrusive_adapter, key_adapter,
 };
 use parking_lot::RwLock;
 use tokio::sync::RwLock as AsyncRwLock;
@@ -38,12 +38,10 @@ where
 {
     link: L,
     id: RegionId,
-    priority: usize,
 }
 
 intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEpItem<L> { link: L } where L: Link }
 key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
-priority_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { priority: usize } where L: Link }
 
 /// Manager of regions and buffer pools.
 ///
@@ -107,7 +105,6 @@ where
             let item = Arc::new(RegionEpItem {
                 link: EL::default(),
                 id,
-                priority: 0,
             });
 
             regions.push(region);
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index b6bfcab0..473fe962 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -1190,9 +1190,7 @@ pub mod tests {
             vec![recorder.clone()];
 
         let config = TestStoreConfig {
-            eviction_config: FifoConfig {
-                segment_ratios: vec![1],
-            },
+            eviction_config: FifoConfig,
             device_config: FsDeviceConfig {
                 dir: PathBuf::from(tempdir.path()),
                 capacity: 16 * MB,
@@ -1242,9 +1240,7 @@ pub mod tests {
         drop(store);
 
         let config = TestStoreConfig {
-            eviction_config: FifoConfig {
-                segment_ratios: vec![1],
-            },
+            eviction_config: FifoConfig,
             device_config: FsDeviceConfig {
                 dir: PathBuf::from(tempdir.path()),
                 capacity: 16 * MB,

From 6bd19cb022ab63ae6e20a5f488693dc329b10bf0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 1 Aug 2023 23:39:44 +0800
Subject: [PATCH 073/261] fix: weight check (#98)

SLEEP REALLY MATTERS!!

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 473fe962..c4e88d7a 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -619,7 +619,7 @@ where
         }
 
         let serialized_len = self.serialized_len(key, &value);
-        if key.serialized_len() + value.serialized_len() == writer.weight {
+        if key.serialized_len() + value.serialized_len() != writer.weight {
             tracing::error!(
                 "weight != key.serialized_len() + value.serialized_len(), weight: {}, key size: {}, value size: {}, key: {:?}",
                 writer.weight, key.serialized_len(), value.serialized_len(), key

From 0e5a595c98c62d9de6f4c76016f2de221a6309e7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 2 Aug 2023 00:11:33 +0800
Subject: [PATCH 074/261] feat: remove index when storage lookup miss (#99)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index c4e88d7a..3a78141c 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -432,6 +432,8 @@ where
         let slice = match region.load(start..end, index.version).await? {
             Some(slice) => slice,
             None => {
+                // Remove index if the storage layer fails to lookup it (because of region version mismatch).
+                self.indices.remove(key);
                 self.metrics
                     .latency_lookup_miss
                     .observe(now.elapsed().as_secs_f64());
@@ -442,7 +444,11 @@ where
 
         let res = match read_entry::<K, V>(slice.as_ref()) {
             Some((_key, value)) => Ok(Some(value)),
-            None => Ok(None),
+            None => {
+                // Remove index if the storage layer fails to lookup it (because of region version mismatch).
+                self.indices.remove(key);
+                Ok(None)
+            }
         };
         slice.destroy().await;
 

From 049f53843c71d191fd0cdb94a9e132c9cf577a36 Mon Sep 17 00:00:00 2001
From: William Wen <44139337+wenym1@users.noreply.github.com>
Date: Thu, 3 Aug 2023 15:09:50 +0800
Subject: [PATCH 075/261] refactor: simplify pointer trait and remove
 DefaultPointerOps (#100)

* refactor: simplify some generic type definition

* refactor: simplify pointer trait and remove DefaultPointerOps

* run clippy

* fix test compile

* rename PointerOps to Pointer

* use adapter in eviction policy associated type
---
 foyer-intrusive/src/collections/dlist.rs      |  56 ++--
 .../src/collections/duplicated_hashmap.rs     |  18 +-
 foyer-intrusive/src/collections/hashmap.rs    |  34 +--
 foyer-intrusive/src/core/adapter.rs           |  47 +--
 foyer-intrusive/src/core/pointer.rs           | 277 +++++++-----------
 foyer-intrusive/src/eviction/fifo.rs          |  61 ++--
 foyer-intrusive/src/eviction/lfu.rs           |  63 ++--
 foyer-intrusive/src/eviction/lru.rs           |  63 ++--
 foyer-intrusive/src/eviction/mod.rs           |  29 +-
 foyer-intrusive/src/eviction/sfifo.rs         |  61 ++--
 10 files changed, 292 insertions(+), 417 deletions(-)

diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index 1d644efb..e40a0695 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -16,7 +16,7 @@ use std::ptr::NonNull;
 
 use crate::core::{
     adapter::{Adapter, Link},
-    pointer::PointerOps,
+    pointer::Pointer,
 };
 
 #[derive(Debug, Default)]
@@ -90,7 +90,7 @@ where
         }
     }
 
-    pub fn front(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+    pub fn front(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         unsafe {
             self.head
                 .map(|link| self.adapter.link2item(link.as_ptr()))
@@ -98,7 +98,7 @@ where
         }
     }
 
-    pub fn back(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+    pub fn back(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         unsafe {
             self.tail
                 .map(|link| self.adapter.link2item(link.as_ptr()))
@@ -106,7 +106,7 @@ where
         }
     }
 
-    pub fn front_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+    pub fn front_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         unsafe {
             self.head
                 .map(|link| self.adapter.link2item(link.as_ptr()))
@@ -115,7 +115,7 @@ where
         }
     }
 
-    pub fn back_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+    pub fn back_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         unsafe {
             self.tail
                 .map(|link| self.adapter.link2item(link.as_ptr()))
@@ -124,21 +124,21 @@ where
         }
     }
 
-    pub fn push_front(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    pub fn push_front(&mut self, ptr: A::Pointer) {
         self.iter_mut().insert_after(ptr);
     }
 
-    pub fn push_back(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    pub fn push_back(&mut self, ptr: A::Pointer) {
         self.iter_mut().insert_before(ptr);
     }
 
-    pub fn pop_front(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+    pub fn pop_front(&mut self) -> Option<A::Pointer> {
         let mut iter = self.iter_mut();
         iter.next();
         iter.remove()
     }
 
-    pub fn pop_back(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+    pub fn pop_back(&mut self) -> Option<A::Pointer> {
         let mut iter = self.iter_mut();
         iter.prev();
         iter.remove()
@@ -224,7 +224,7 @@ where
         self.link.is_some()
     }
 
-    pub fn get(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+    pub fn get(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         self.link
             .map(|link| unsafe { &*self.dlist.adapter.link2item(link.as_ptr()) })
     }
@@ -290,12 +290,12 @@ where
         self.link.is_some()
     }
 
-    pub fn get(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+    pub fn get(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         self.link
             .map(|link| unsafe { &*self.dlist.adapter.link2item(link.as_ptr()) })
     }
 
-    pub fn get_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+    pub fn get_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         self.link
             .map(|link| unsafe { &mut *(self.dlist.adapter.link2item(link.as_ptr()) as *mut _) })
     }
@@ -337,7 +337,7 @@ where
     }
 
     /// Removes the current item from [`DList`] and move next.
-    pub fn remove(&mut self) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+    pub fn remove(&mut self) -> Option<A::Pointer> {
         unsafe {
             if !self.is_valid() {
                 return None;
@@ -346,7 +346,7 @@ where
             let mut link = self.link.unwrap();
 
             let item = self.dlist.adapter.link2item(link.as_ptr());
-            let ptr = self.dlist.adapter.pointer_ops().from_raw(item);
+            let ptr = A::Pointer::from_raw(item);
 
             // fix head and tail if node is either of that
             let mut prev = link.as_ref().prev;
@@ -381,9 +381,9 @@ where
     /// Link a new ptr before the current one.
     ///
     /// If iter is on null, link to tail.
-    pub fn insert_before(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    pub fn insert_before(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = self.dlist.adapter.pointer_ops().into_raw(ptr);
+            let item_new = A::Pointer::into_raw(ptr);
             let mut link_new =
                 NonNull::new_unchecked(self.dlist.adapter.item2link(item_new) as *mut A::Link);
             assert!(!link_new.as_ref().is_linked());
@@ -409,9 +409,9 @@ where
     /// Link a new ptr after the current one.
     ///
     /// If iter is on null, link to head.
-    pub fn insert_after(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    pub fn insert_after(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = self.dlist.adapter.pointer_ops().into_raw(ptr);
+            let item_new = A::Pointer::into_raw(ptr);
             let mut link_new =
                 NonNull::new_unchecked(self.dlist.adapter.item2link(item_new) as *mut A::Link);
             assert!(!link_new.as_ref().is_linked());
@@ -471,7 +471,7 @@ impl<'a, A> Iterator for DListIter<'a, A>
 where
     A: Adapter<Link = DListLink>,
 {
-    type Item = &'a <A::PointerOps as PointerOps>::Item;
+    type Item = &'a <A::Pointer as Pointer>::Item;
 
     fn next(&mut self) -> Option<Self::Item> {
         self.next();
@@ -486,7 +486,7 @@ impl<'a, A> Iterator for DListIterMut<'a, A>
 where
     A: Adapter<Link = DListLink>,
 {
-    type Item = &'a mut <A::PointerOps as PointerOps>::Item;
+    type Item = &'a mut <A::Pointer as Pointer>::Item;
 
     fn next(&mut self) -> Option<Self::Item> {
         self.next();
@@ -507,7 +507,7 @@ mod tests {
 
     use itertools::Itertools;
 
-    use crate::{core::pointer::DefaultPointerOps, intrusive_adapter};
+    use crate::intrusive_adapter;
 
     use super::*;
 
@@ -527,31 +527,27 @@ mod tests {
     }
 
     #[derive(Debug, Default)]
-    struct DListAdapter(DefaultPointerOps<Box<DListItem>>);
+    struct DListAdapter;
 
     unsafe impl Adapter for DListAdapter {
-        type PointerOps = DefaultPointerOps<Box<DListItem>>;
+        type Pointer = Box<DListItem>;
 
         type Link = DListLink;
 
         fn new() -> Self {
-            Self::default()
-        }
-
-        fn pointer_ops(&self) -> &Self::PointerOps {
-            &self.0
+            Self
         }
 
         unsafe fn link2item(
             &self,
             link: *const Self::Link,
-        ) -> *const <Self::PointerOps as PointerOps>::Item {
+        ) -> *const <Self::Pointer as Pointer>::Item {
             crate::container_of!(link, DListItem, link)
         }
 
         unsafe fn item2link(
             &self,
-            item: *const <Self::PointerOps as PointerOps>::Item,
+            item: *const <Self::Pointer as Pointer>::Item,
         ) -> *const Self::Link {
             (item as *const u8).add(crate::offset_of!(DListItem, link)) as *const _
         }
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index ffdbb5c3..f70affa3 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -21,7 +21,7 @@ use twox_hash::XxHash64;
 use crate::{
     core::{
         adapter::{Adapter, KeyAdapter, Link},
-        pointer::PointerOps,
+        pointer::Pointer,
     },
     intrusive_adapter,
 };
@@ -104,7 +104,7 @@ where
                     while iter_group.is_valid() {
                         let link_group = iter_group.remove().unwrap();
                         let item = self.adapter.link2item(link_group.as_ptr());
-                        let _ = self.adapter.pointer_ops().from_raw(item);
+                        let _ = A::Pointer::from_raw(item);
                     }
                 }
             }
@@ -131,9 +131,9 @@ where
         }
     }
 
-    pub fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    pub fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = self.adapter.pointer_ops().into_raw(ptr);
+            let item_new = A::Pointer::into_raw(ptr);
             let mut link_new =
                 NonNull::new_unchecked(self.adapter.item2link(item_new) as *mut A::Link);
 
@@ -158,7 +158,7 @@ where
         }
     }
 
-    pub fn remove(&mut self, key: &K) -> Vec<<A::PointerOps as PointerOps>::Pointer> {
+    pub fn remove(&mut self, key: &K) -> Vec<A::Pointer> {
         unsafe {
             let hash = self.hash_key(key);
             let slot = (self.slots.len() - 1) & hash as usize;
@@ -173,7 +173,7 @@ where
                         let link = iter.remove().unwrap();
                         debug_assert!(!link.as_ref().is_linked());
                         let item = self.adapter.link2item(link.as_ptr());
-                        let ptr = self.adapter.pointer_ops().from_raw(item);
+                        let ptr = A::Pointer::from_raw(item);
                         res.push(ptr);
                     }
                     debug_assert!(link.as_ref().group.is_empty());
@@ -185,7 +185,7 @@ where
         }
     }
 
-    pub fn lookup(&self, key: &K) -> Vec<&<A::PointerOps as PointerOps>::Item> {
+    pub fn lookup(&self, key: &K) -> Vec<&<A::Pointer as Pointer>::Item> {
         unsafe {
             let hash = self.hash_key(key);
             let slot = (self.slots.len() - 1) & hash as usize;
@@ -223,7 +223,7 @@ where
     pub unsafe fn remove_in_place(
         &mut self,
         mut link: NonNull<DuplicatedHashMapLink>,
-    ) -> <A::PointerOps as PointerOps>::Pointer {
+    ) -> A::Pointer {
         assert!(link.as_ref().is_linked());
         let item = self.adapter.link2item(link.as_ptr());
         let key = &*self.adapter.item2key(item);
@@ -276,7 +276,7 @@ where
         debug_assert!(!link.as_ref().group_link.is_linked());
         debug_assert!(link.as_ref().group.is_empty());
 
-        self.adapter.pointer_ops().from_raw(item)
+        A::Pointer::from_raw(item)
     }
 
     /// # Safety
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index f6d8f871..0f50c4bc 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -21,7 +21,7 @@ use twox_hash::XxHash64;
 use crate::{
     core::{
         adapter::{KeyAdapter, Link},
-        pointer::PointerOps,
+        pointer::Pointer,
     },
     intrusive_adapter,
 };
@@ -79,7 +79,7 @@ where
                 while iter.is_valid() {
                     let link = iter.remove().unwrap();
                     let item = self.adapter.link2item(link.as_ptr());
-                    let _ = self.adapter.pointer_ops().from_raw(item);
+                    let _ = A::Pointer::from_raw(item);
                 }
             }
         }
@@ -105,12 +105,9 @@ where
         }
     }
 
-    pub fn insert(
-        &mut self,
-        ptr: <A::PointerOps as PointerOps>::Pointer,
-    ) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+    pub fn insert(&mut self, ptr: A::Pointer) -> Option<A::Pointer> {
         unsafe {
-            let item_new = self.adapter.pointer_ops().into_raw(ptr);
+            let item_new = A::Pointer::into_raw(ptr);
             let link_new = NonNull::new_unchecked(self.adapter.item2link(item_new) as *mut A::Link);
 
             let key_new = &*self.adapter.item2key(item_new);
@@ -130,7 +127,7 @@ where
         }
     }
 
-    pub fn remove(&mut self, key: &K) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+    pub fn remove(&mut self, key: &K) -> Option<A::Pointer> {
         unsafe {
             let hash = self.hash_key(key);
             let slot = (self.slots.len() - 1) & hash as usize;
@@ -143,7 +140,7 @@ where
         }
     }
 
-    pub fn lookup(&self, key: &K) -> Option<&<A::PointerOps as PointerOps>::Item> {
+    pub fn lookup(&self, key: &K) -> Option<&<A::Pointer as Pointer>::Item> {
         unsafe {
             let hash = self.hash_key(key);
             let slot = (self.slots.len() - 1) & hash as usize;
@@ -174,10 +171,7 @@ where
     /// # Safety
     ///
     /// `link` MUST be in this [`HashMap`].
-    pub unsafe fn remove_in_place(
-        &mut self,
-        link: NonNull<HashMapLink>,
-    ) -> <A::PointerOps as PointerOps>::Pointer {
+    pub unsafe fn remove_in_place(&mut self, link: NonNull<HashMapLink>) -> A::Pointer {
         assert!(link.as_ref().is_linked());
         let item = self.adapter.link2item(link.as_ptr());
         let key = &*self.adapter.item2key(item);
@@ -187,7 +181,7 @@ where
             .iter_mut_from_raw(link.as_ref().dlist_link.raw())
             .remove();
         self.len -= 1;
-        self.adapter.pointer_ops().from_raw(item)
+        A::Pointer::from_raw(item)
     }
 
     /// # Safety
@@ -235,16 +229,12 @@ where
     /// # Safety
     ///
     /// there must be at most one matches in the slot
-    unsafe fn remove_inner(
-        &mut self,
-        key: &K,
-        slot: usize,
-    ) -> Option<<A::PointerOps as PointerOps>::Pointer> {
+    unsafe fn remove_inner(&mut self, key: &K, slot: usize) -> Option<A::Pointer> {
         match self.lookup_inner_mut(key, slot) {
             Some(mut iter) => {
                 let link = iter.remove().unwrap();
                 let item = self.adapter.link2item(link.as_ptr());
-                let ptr = self.adapter.pointer_ops().from_raw(item);
+                let ptr = A::Pointer::from_raw(item);
                 Some(ptr)
             }
             None => None,
@@ -296,13 +286,13 @@ where
         self.iters[self.slot].is_valid()
     }
 
-    pub fn get(&self) -> Option<&<A::PointerOps as PointerOps>::Item> {
+    pub fn get(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         self.iters[self.slot]
             .get()
             .map(|link| unsafe { &*(self.adapter.link2item(link.raw().as_ptr()) as *const _) })
     }
 
-    pub fn get_mut(&mut self) -> Option<&mut <A::PointerOps as PointerOps>::Item> {
+    pub fn get_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         self.iters[self.slot]
             .get()
             .map(|link| unsafe { &mut *(self.adapter.link2item(link.raw().as_ptr()) as *mut _) })
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 00534096..f226013e 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -30,7 +30,7 @@ use std::fmt::Debug;
 
 use foyer_common::code::Key;
 
-use crate::core::pointer::PointerOps;
+use crate::core::pointer::Pointer;
 
 pub trait Link: Send + Sync + 'static + Default + Debug {
     fn is_linked(&self) -> bool;
@@ -42,28 +42,20 @@ pub trait Link: Send + Sync + 'static + Default + Debug {
 ///
 /// [`Adapter`] is recommanded to be generated by macro `instrusive_adapter!`.
 pub unsafe trait Adapter: Send + Sync + 'static {
-    type PointerOps: PointerOps;
+    type Pointer: Pointer;
     type Link: Link;
 
     fn new() -> Self;
 
-    fn pointer_ops(&self) -> &Self::PointerOps;
-
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn link2item(
-        &self,
-        link: *const Self::Link,
-    ) -> *const <Self::PointerOps as PointerOps>::Item;
+    unsafe fn link2item(&self, link: *const Self::Link) -> *const <Self::Pointer as Pointer>::Item;
 
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn item2link(
-        &self,
-        item: *const <Self::PointerOps as PointerOps>::Item,
-    ) -> *const Self::Link;
+    unsafe fn item2link(&self, item: *const <Self::Pointer as Pointer>::Item) -> *const Self::Link;
 }
 
 /// # Safety
@@ -77,10 +69,7 @@ pub unsafe trait KeyAdapter: Adapter {
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn item2key(
-        &self,
-        item: *const <Self::PointerOps as PointerOps>::Item,
-    ) -> *const Self::Key;
+    unsafe fn item2key(&self, item: *const <Self::Pointer as Pointer>::Item) -> *const Self::Key;
 }
 
 /// # Safety
@@ -96,7 +85,7 @@ pub unsafe trait PriorityAdapter: Adapter {
     /// Pointer operations MUST be valid.
     unsafe fn item2priority(
         &self,
-        item: *const <Self::PointerOps as PointerOps>::Item,
+        item: *const <Self::Pointer as Pointer>::Item,
     ) -> *const Self::Priority;
 }
 
@@ -121,7 +110,7 @@ pub unsafe trait PriorityAdapter: Adapter {
 /// ```
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
-/// use foyer_intrusive::core::pointer::PointerOps;
+/// use foyer_intrusive::core::pointer::Pointer;
 /// use foyer_intrusive::eviction::EvictionPolicy;
 /// use std::sync::Arc;
 ///
@@ -143,38 +132,32 @@ macro_rules! intrusive_adapter {
         $vis:vis $name:ident ($($args:tt),*) = $pointer:ty: $item:path { $field:ident: $link:ty } $($where_:tt)*
     ) => {
         $vis struct $name<$($args),*> $($where_)* {
-            pointer_ops: $crate::core::pointer::DefaultPointerOps<$pointer>,
-            _marker: std::marker::PhantomData<($($args),*)>
+            _marker: std::marker::PhantomData<($pointer, $($args),*)>
         }
 
         unsafe impl<$($args),*> Send for $name<$($args),*> $($where_)* {}
         unsafe impl<$($args),*> Sync for $name<$($args),*> $($where_)* {}
 
         unsafe impl<$($args),*> $crate::core::adapter::Adapter for $name<$($args),*> $($where_)*{
-            type PointerOps = $crate::core::pointer::DefaultPointerOps<$pointer>;
+            type Pointer = $pointer;
             type Link = $link;
 
             fn new() -> Self {
                 Self {
-                    pointer_ops: Default::default(),
                     _marker: std::marker::PhantomData,
                 }
             }
 
-            fn pointer_ops(&self) -> &Self::PointerOps {
-                &self.pointer_ops
-            }
-
             unsafe fn link2item(
                 &self,
                 link: *const Self::Link,
-            ) -> *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item {
+            ) -> *const <Self::Pointer as $crate::core::pointer::Pointer>::Item {
                 $crate::container_of!(link, $item, $field)
             }
 
             unsafe fn item2link(
                 &self,
-                item: *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item,
+                item: *const <Self::Pointer as $crate::core::pointer::Pointer>::Item,
             ) -> *const Self::Link {
                 (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
             }
@@ -217,7 +200,7 @@ macro_rules! intrusive_adapter {
 /// ```
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
-/// use foyer_intrusive::core::pointer::PointerOps;
+/// use foyer_intrusive::core::pointer::Pointer;
 /// use foyer_intrusive::eviction::EvictionPolicy;
 /// use std::sync::Arc;
 ///
@@ -243,7 +226,7 @@ macro_rules! key_adapter {
 
             unsafe fn item2key(
                 &self,
-                item: *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item,
+                item: *const <Self::Pointer as $crate::core::pointer::Pointer>::Item,
             ) -> *const Self::Key {
                 (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
             }
@@ -286,7 +269,7 @@ macro_rules! key_adapter {
 /// ```
 /// use foyer_intrusive::{intrusive_adapter, priority_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, PriorityAdapter, Link};
-/// use foyer_intrusive::core::pointer::PointerOps;
+/// use foyer_intrusive::core::pointer::Pointer;
 /// use foyer_intrusive::eviction::EvictionPolicy;
 /// use std::sync::Arc;
 ///
@@ -312,7 +295,7 @@ macro_rules! priority_adapter {
 
             unsafe fn item2priority(
                 &self,
-                item: *const <Self::PointerOps as $crate::core::pointer::PointerOps>::Item,
+                item: *const <Self::Pointer as $crate::core::pointer::Pointer>::Item,
             ) -> *const Self::Priority {
                 (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
             }
diff --git a/foyer-intrusive/src/core/pointer.rs b/foyer-intrusive/src/core/pointer.rs
index 20daacef..01f7c079 100644
--- a/foyer-intrusive/src/core/pointer.rs
+++ b/foyer-intrusive/src/core/pointer.rs
@@ -26,252 +26,215 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, pin::Pin, ptr::NonNull, rc::Rc, sync::Arc};
+use std::{fmt::Debug, pin::Pin, ptr::NonNull, rc::Rc, sync::Arc};
 
 /// # Safety
 ///
 /// Pointer operations MUST be valid.
-pub unsafe trait PointerOps {
+pub unsafe trait Pointer {
     type Item: ?Sized;
-    type Pointer: Debug;
 
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn from_raw(&self, item: *const Self::Item) -> Self::Pointer;
+    unsafe fn from_raw(item: *const Self::Item) -> Self;
 
-    fn into_raw(&self, ptr: Self::Pointer) -> *const Self::Item;
+    fn into_raw(self) -> *const Self::Item;
 
-    fn as_ptr(&self, ptr: &Self::Pointer) -> *const Self::Item;
+    fn as_ptr(&self) -> *const Self::Item;
 }
 
 /// # Safety
 ///
 /// Pointer operations MUST be valid.
-pub unsafe trait DowngradablePointerOps: PointerOps {
+pub unsafe trait DowngradablePointerOps: Pointer {
     type WeakPointer;
 
-    fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer;
+    fn downgrade(&self) -> Self::WeakPointer;
 }
 
-#[derive(Debug)]
-pub struct DefaultPointerOps<Pointer>(PhantomData<Pointer>);
-
-impl<Pointer> DefaultPointerOps<Pointer> {
-    /// Constructs an instance of `DefaultPointerOps`.
-    #[inline]
-    pub const fn new() -> DefaultPointerOps<Pointer> {
-        DefaultPointerOps(PhantomData)
-    }
-}
-
-impl<Pointer> Clone for DefaultPointerOps<Pointer> {
-    #[inline]
-    fn clone(&self) -> Self {
-        *self
-    }
-}
-
-impl<Pointer> Copy for DefaultPointerOps<Pointer> {}
-
-impl<Pointer> Default for DefaultPointerOps<Pointer> {
-    #[inline]
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-unsafe impl<'a, T: ?Sized + Debug> PointerOps for DefaultPointerOps<&'a T> {
+unsafe impl<'a, T: ?Sized + Debug> Pointer for &'a T {
     type Item = T;
-    type Pointer = &'a T;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> &'a T {
+    unsafe fn from_raw(raw: *const T) -> &'a T {
         &*raw
     }
 
     #[inline]
-    fn into_raw(&self, ptr: &'a T) -> *const T {
-        ptr
+    fn into_raw(self) -> *const T {
+        self
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &&'a T) -> *const T {
-        *ptr as *const _
+    fn as_ptr(&self) -> *const T {
+        *self as *const _
     }
 }
 
-unsafe impl<'a, T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<&'a T>> {
+unsafe impl<'a, T: ?Sized + Debug> Pointer for Pin<&'a T> {
     type Item = T;
-    type Pointer = Pin<&'a T>;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> Pin<&'a T> {
+    unsafe fn from_raw(raw: *const T) -> Pin<&'a T> {
         Pin::new_unchecked(&*raw)
     }
 
     #[inline]
-    fn into_raw(&self, ptr: Pin<&'a T>) -> *const T {
-        unsafe { Pin::into_inner_unchecked(ptr) as *const T }
+    fn into_raw(self) -> *const T {
+        unsafe { Pin::into_inner_unchecked(self) as *const T }
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &Pin<&'a T>) -> *const T {
-        ptr.get_ref() as *const _
+    fn as_ptr(&self) -> *const T {
+        self.get_ref() as *const _
     }
 }
 
-unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<NonNull<T>> {
+unsafe impl<T: ?Sized + Debug> Pointer for NonNull<T> {
     type Item = T;
-    type Pointer = NonNull<T>;
 
-    unsafe fn from_raw(&self, raw: *const T) -> NonNull<T> {
+    unsafe fn from_raw(raw: *const T) -> NonNull<T> {
         NonNull::new_unchecked(raw as *mut _)
     }
 
-    fn into_raw(&self, ptr: NonNull<T>) -> *const T {
-        ptr.as_ptr()
+    fn into_raw(self) -> *const T {
+        self.as_ptr()
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &NonNull<T>) -> *const T {
-        ptr.as_ptr()
+    fn as_ptr(&self) -> *const T {
+        NonNull::as_ptr(*self)
     }
 }
 
-unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Box<T>> {
+unsafe impl<T: ?Sized + Debug> Pointer for Box<T> {
     type Item = T;
-    type Pointer = Box<T>;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> Box<T> {
+    unsafe fn from_raw(raw: *const T) -> Box<T> {
         Box::from_raw(raw as *mut T)
     }
 
     #[inline]
-    fn into_raw(&self, ptr: Box<T>) -> *const T {
-        Box::into_raw(ptr) as *const T
+    fn into_raw(self) -> *const T {
+        Box::into_raw(self) as *const T
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &Box<T>) -> *const T {
-        ptr.as_ref() as *const _
+    fn as_ptr(&self) -> *const T {
+        self.as_ref() as *const _
     }
 }
 
-unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<Box<T>>> {
+unsafe impl<T: ?Sized + Debug> Pointer for Pin<Box<T>> {
     type Item = T;
-    type Pointer = Pin<Box<T>>;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> Pin<Box<T>> {
+    unsafe fn from_raw(raw: *const T) -> Pin<Box<T>> {
         Pin::new_unchecked(Box::from_raw(raw as *mut T))
     }
 
     #[inline]
-    fn into_raw(&self, ptr: Pin<Box<T>>) -> *const T {
-        Box::into_raw(unsafe { Pin::into_inner_unchecked(ptr) }) as *const T
+    fn into_raw(self) -> *const T {
+        Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as *const T
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &Pin<Box<T>>) -> *const T {
-        ptr.as_ref().get_ref() as *const _
+    fn as_ptr(&self) -> *const T {
+        self.as_ref().get_ref() as *const _
     }
 }
 
-unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Rc<T>> {
+unsafe impl<T: ?Sized + Debug> Pointer for Rc<T> {
     type Item = T;
-    type Pointer = Rc<T>;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> Rc<T> {
+    unsafe fn from_raw(raw: *const T) -> Rc<T> {
         Rc::from_raw(raw)
     }
 
     #[inline]
-    fn into_raw(&self, ptr: Rc<T>) -> *const T {
-        Rc::into_raw(ptr)
+    fn into_raw(self) -> *const T {
+        Rc::into_raw(self)
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &Rc<T>) -> *const T {
-        Rc::as_ptr(ptr)
+    fn as_ptr(&self) -> *const T {
+        Rc::as_ptr(self)
     }
 }
 
-unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<Rc<T>>> {
+unsafe impl<T: ?Sized + Debug> Pointer for Pin<Rc<T>> {
     type Item = T;
-    type Pointer = Pin<Rc<T>>;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> Pin<Rc<T>> {
+    unsafe fn from_raw(raw: *const T) -> Pin<Rc<T>> {
         Pin::new_unchecked(Rc::from_raw(raw))
     }
 
     #[inline]
-    fn into_raw(&self, ptr: Pin<Rc<T>>) -> *const T {
-        Rc::into_raw(unsafe { Pin::into_inner_unchecked(ptr) })
+    fn into_raw(self) -> *const T {
+        Rc::into_raw(unsafe { Pin::into_inner_unchecked(self) })
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &Pin<Rc<T>>) -> *const T {
-        ptr.as_ref().get_ref() as *const _
+    fn as_ptr(&self) -> *const T {
+        self.as_ref().get_ref() as *const _
     }
 }
 
-unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Arc<T>> {
+unsafe impl<T: ?Sized + Debug> Pointer for Arc<T> {
     type Item = T;
-    type Pointer = Arc<T>;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> Arc<T> {
+    unsafe fn from_raw(raw: *const T) -> Arc<T> {
         Arc::from_raw(raw)
     }
 
     #[inline]
-    fn into_raw(&self, ptr: Arc<T>) -> *const T {
-        Arc::into_raw(ptr)
+    fn into_raw(self) -> *const T {
+        Arc::into_raw(self)
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &Arc<T>) -> *const T {
-        Arc::as_ptr(ptr)
+    fn as_ptr(&self) -> *const T {
+        Arc::as_ptr(self)
     }
 }
 
-unsafe impl<T: ?Sized + Debug> PointerOps for DefaultPointerOps<Pin<Arc<T>>> {
+unsafe impl<T: ?Sized + Debug> Pointer for Pin<Arc<T>> {
     type Item = T;
-    type Pointer = Pin<Arc<T>>;
 
     #[inline]
-    unsafe fn from_raw(&self, raw: *const T) -> Pin<Arc<T>> {
+    unsafe fn from_raw(raw: *const T) -> Pin<Arc<T>> {
         Pin::new_unchecked(Arc::from_raw(raw))
     }
 
     #[inline]
-    fn into_raw(&self, ptr: Pin<Arc<T>>) -> *const T {
-        Arc::into_raw(unsafe { Pin::into_inner_unchecked(ptr) })
+    fn into_raw(self) -> *const T {
+        Arc::into_raw(unsafe { Pin::into_inner_unchecked(self) })
     }
 
     #[inline]
-    fn as_ptr(&self, ptr: &Pin<Arc<T>>) -> *const T {
-        ptr.as_ref().get_ref() as *const _
+    fn as_ptr(&self) -> *const T {
+        self.as_ref().get_ref() as *const _
     }
 }
 
-unsafe impl<T: ?Sized + Debug> DowngradablePointerOps for DefaultPointerOps<Rc<T>> {
+unsafe impl<T: ?Sized + Debug> DowngradablePointerOps for Rc<T> {
     type WeakPointer = std::rc::Weak<T>;
 
-    fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer {
-        Rc::downgrade(ptr)
+    fn downgrade(&self) -> Self::WeakPointer {
+        Rc::downgrade(self)
     }
 }
 
-unsafe impl<T: ?Sized + Debug> DowngradablePointerOps for DefaultPointerOps<Arc<T>> {
+unsafe impl<T: ?Sized + Debug> DowngradablePointerOps for Arc<T> {
     type WeakPointer = std::sync::Weak<T>;
 
-    fn downgrade(&self, ptr: &<Self as PointerOps>::Pointer) -> Self::WeakPointer {
-        Arc::downgrade(ptr)
+    fn downgrade(&self) -> Self::WeakPointer {
+        Arc::downgrade(self)
     }
 }
 
@@ -280,43 +243,33 @@ mod tests {
     use super::*;
     use std::{boxed::Box, fmt::Debug, mem, pin::Pin, rc::Rc, sync::Arc};
 
-    /// Clones a `PointerOps::Pointer` from a `*const PointerOps::Value`
+    /// Clones a `Pointer` from a `*const Pointer::Value`
     ///
     /// This method is only safe to call if the raw pointer is known to be
-    /// managed by the provided `PointerOps` type.
+    /// managed by the provided `Pointer` type.
     #[inline]
-    unsafe fn clone_pointer_from_raw<T: PointerOps>(
-        pointer_ops: &T,
-        ptr: *const T::Item,
-    ) -> T::Pointer
-    where
-        T::Pointer: Clone,
-    {
+    unsafe fn clone_pointer_from_raw<T: Pointer + Clone>(ptr: *const T::Item) -> T {
         use std::{mem::ManuallyDrop, ops::Deref};
 
         /// Guard which converts an pointer back into its raw version
         /// when it gets dropped. This makes sure we also perform a full
         /// `from_raw` and `into_raw` round trip - even in the case of panics.
-        struct PointerGuard<'a, T: PointerOps> {
-            pointer: ManuallyDrop<T::Pointer>,
-            pointer_ops: &'a T,
+        struct PointerGuard<T: Pointer> {
+            pointer: ManuallyDrop<T>,
         }
 
-        impl<'a, T: PointerOps> Drop for PointerGuard<'a, T> {
+        impl<T: Pointer> Drop for PointerGuard<T> {
             #[inline]
             fn drop(&mut self) {
                 // Prevent shared pointers from being released by converting them
                 // back into the raw pointers
                 // SAFETY: `pointer` is never dropped. `ManuallyDrop::take` is not stable until 1.42.0.
-                let _ = self
-                    .pointer_ops
-                    .into_raw(unsafe { core::ptr::read(&*self.pointer) });
+                let _ = T::into_raw(unsafe { core::ptr::read(&*self.pointer) });
             }
         }
 
         let holder = PointerGuard {
-            pointer: ManuallyDrop::new(pointer_ops.from_raw(ptr)),
-            pointer_ops,
+            pointer: ManuallyDrop::new(T::from_raw(ptr)),
         };
         holder.pointer.deref().clone()
     }
@@ -324,12 +277,11 @@ mod tests {
     #[test]
     fn test_box() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Box<_>>::new();
             let p = Box::new(1);
             let a: *const i32 = &*p;
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
-            let p2: Box<i32> = pointer_ops.from_raw(r);
+            let p2: Box<i32> = <Box<i32> as Pointer>::from_raw(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -338,12 +290,11 @@ mod tests {
     #[test]
     fn test_rc() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Rc<_>>::new();
             let p = Rc::new(1);
             let a: *const i32 = &*p;
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
-            let p2: Rc<i32> = pointer_ops.from_raw(r);
+            let p2: Rc<i32> = <Rc<_> as Pointer>::from_raw(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -352,12 +303,11 @@ mod tests {
     #[test]
     fn test_arc() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Arc<_>>::new();
             let p = Arc::new(1);
             let a: *const i32 = &*p;
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
-            let p2: Arc<i32> = pointer_ops.from_raw(r);
+            let p2: Arc<i32> = <Arc<_> as Pointer>::from_raw(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -366,14 +316,13 @@ mod tests {
     #[test]
     fn test_box_unsized() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Box<_>>::new();
             let p = Box::new(1) as Box<dyn Debug>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Box<dyn Debug> = pointer_ops.from_raw(r);
+            let p2: Box<dyn Debug> = <Box<_> as Pointer>::from_raw(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -383,14 +332,13 @@ mod tests {
     #[test]
     fn test_rc_unsized() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Rc<_>>::new();
             let p = Rc::new(1) as Rc<dyn Debug>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Rc<dyn Debug> = pointer_ops.from_raw(r);
+            let p2: Rc<dyn Debug> = <Rc<_> as Pointer>::from_raw(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -400,14 +348,13 @@ mod tests {
     #[test]
     fn test_arc_unsized() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Arc<_>>::new();
             let p = Arc::new(1) as Arc<dyn Debug>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Arc<dyn Debug> = pointer_ops.from_raw(r);
+            let p2: Arc<dyn Debug> = <Arc<_> as Pointer>::from_raw(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -417,10 +364,9 @@ mod tests {
     #[test]
     fn clone_arc_from_raw() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Arc<_>>::new();
             let p = Arc::new(1);
             let raw = Arc::as_ptr(&p);
-            let p2: Arc<i32> = clone_pointer_from_raw(&pointer_ops, raw);
+            let p2: Arc<i32> = clone_pointer_from_raw(raw);
             assert_eq!(2, Arc::strong_count(&p2));
         }
     }
@@ -428,10 +374,9 @@ mod tests {
     #[test]
     fn clone_rc_from_raw() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Rc<_>>::new();
             let p = Rc::new(1);
             let raw = Rc::as_ptr(&p);
-            let p2: Rc<i32> = clone_pointer_from_raw(&pointer_ops, raw);
+            let p2: Rc<i32> = clone_pointer_from_raw(raw);
             assert_eq!(2, Rc::strong_count(&p2));
         }
     }
@@ -439,12 +384,11 @@ mod tests {
     #[test]
     fn test_pin_box() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Box<_>>>::new();
             let p = Pin::new(Box::new(1));
             let a: *const i32 = &*p;
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
-            let p2: Pin<Box<i32>> = pointer_ops.from_raw(r);
+            let p2: Pin<Box<i32>> = <Pin<Box<_>> as Pointer>::from_raw(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -453,12 +397,11 @@ mod tests {
     #[test]
     fn test_pin_rc() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Rc<_>>>::new();
             let p = Pin::new(Rc::new(1));
             let a: *const i32 = &*p;
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
-            let p2: Pin<Rc<i32>> = pointer_ops.from_raw(r);
+            let p2: Pin<Rc<i32>> = <Pin<Rc<_>> as Pointer>::from_raw(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -467,12 +410,11 @@ mod tests {
     #[test]
     fn test_pin_arc() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Arc<_>>>::new();
             let p = Pin::new(Arc::new(1));
             let a: *const i32 = &*p;
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
-            let p2: Pin<Arc<i32>> = pointer_ops.from_raw(r);
+            let p2: Pin<Arc<i32>> = <Pin<Arc<_>> as Pointer>::from_raw(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -481,14 +423,13 @@ mod tests {
     #[test]
     fn test_pin_box_unsized() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Box<_>>>::new();
             let p = Pin::new(Box::new(1)) as Pin<Box<dyn Debug>>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Pin<Box<dyn Debug>> = pointer_ops.from_raw(r);
+            let p2: Pin<Box<dyn Debug>> = <Pin<Box<_>> as Pointer>::from_raw(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -498,14 +439,13 @@ mod tests {
     #[test]
     fn test_pin_rc_unsized() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Rc<_>>>::new();
             let p = Pin::new(Rc::new(1)) as Pin<Rc<dyn Debug>>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Pin<Rc<dyn Debug>> = pointer_ops.from_raw(r);
+            let p2: Pin<Rc<dyn Debug>> = <Pin<Rc<_>> as Pointer>::from_raw(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -515,14 +455,13 @@ mod tests {
     #[test]
     fn test_pin_arc_unsized() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Arc<_>>>::new();
             let p = Pin::new(Arc::new(1)) as Pin<Arc<dyn Debug>>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = pointer_ops.into_raw(p);
+            let r = p.into_raw();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Pin<Arc<dyn Debug>> = pointer_ops.from_raw(r);
+            let p2: Pin<Arc<dyn Debug>> = <Pin<Arc<_>> as Pointer>::from_raw(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -532,11 +471,10 @@ mod tests {
     #[test]
     fn clone_pin_arc_from_raw() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Arc<_>>>::new();
             let p = Pin::new(Arc::new(1));
-            let raw = pointer_ops.into_raw(p);
-            let p2: Pin<Arc<i32>> = clone_pointer_from_raw(&pointer_ops, raw);
-            let _p = pointer_ops.from_raw(raw);
+            let raw = p.into_raw();
+            let p2: Pin<Arc<i32>> = clone_pointer_from_raw(raw);
+            let _p = <Pin<Arc<_>> as Pointer>::from_raw(raw);
             assert_eq!(2, Arc::strong_count(&Pin::into_inner(p2)));
         }
     }
@@ -544,11 +482,10 @@ mod tests {
     #[test]
     fn clone_pin_rc_from_raw() {
         unsafe {
-            let pointer_ops = DefaultPointerOps::<Pin<Rc<_>>>::new();
             let p = Pin::new(Rc::new(1));
-            let raw = pointer_ops.into_raw(p);
-            let p2: Pin<Rc<i32>> = clone_pointer_from_raw(&pointer_ops, raw);
-            let _p = pointer_ops.from_raw(raw);
+            let raw = p.into_raw();
+            let p2: Pin<Rc<i32>> = clone_pointer_from_raw(raw);
+            let _p = <Pin<Rc<_>> as Pointer>::from_raw(raw);
             assert_eq!(2, Rc::strong_count(&Pin::into_inner(p2)));
         }
     }
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index c788765e..bec8e750 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -32,7 +32,7 @@ use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, Link},
-        pointer::PointerOps,
+        pointer::Pointer,
     },
     intrusive_adapter,
 };
@@ -65,7 +65,7 @@ intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DList
 pub struct Fifo<A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     queue: DList<FifoLinkAdapter>,
 
@@ -79,7 +79,7 @@ where
 impl<A> Drop for Fifo<A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     fn drop(&mut self) {
         let mut to_remove = vec![];
@@ -95,7 +95,7 @@ where
 impl<A> Fifo<A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     pub fn new(config: FifoConfig) -> Self {
         Self {
@@ -109,9 +109,9 @@ where
         }
     }
 
-    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let item = A::Pointer::into_raw(ptr);
             let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
 
             assert!(!link.as_ref().is_linked());
@@ -122,12 +122,9 @@ where
         }
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<A::PointerOps as PointerOps>::Pointer,
-    ) -> <A::PointerOps as PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let item = A::Pointer::as_ptr(ptr);
             let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
 
             assert!(link.as_ref().is_linked());
@@ -139,11 +136,11 @@ where
 
             self.len -= 1;
 
-            self.adapter.pointer_ops().from_raw(item)
+            A::Pointer::from_raw(item)
         }
     }
 
-    fn access(&mut self, _ptr: &<A::PointerOps as PointerOps>::Pointer) {}
+    fn access(&mut self, _ptr: &A::Pointer) {}
 
     fn len(&self) -> usize {
         self.len
@@ -165,33 +162,33 @@ where
 pub struct FifoIter<'a, A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     fifo: &'a Fifo<A>,
     iter: DListIter<'a, FifoLinkAdapter>,
 
-    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+    ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
 
 impl<'a, A> FifoIter<'a, A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     unsafe fn update_ptr(&mut self, link: NonNull<FifoLink>) {
         std::mem::forget(self.ptr.take());
 
         let item = self.fifo.adapter.link2item(link.as_ptr());
-        let ptr = self.fifo.adapter.pointer_ops().from_raw(item);
+        let ptr = A::Pointer::from_raw(item);
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
-    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+    unsafe fn ptr(&self) -> Option<&'a <A as Adapter>::Pointer> {
         if self.ptr.is_none() {
             return None;
         }
         let ptr = self.ptr.as_ref().unwrap();
-        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        let raw = ptr as *const <A as Adapter>::Pointer;
         Some(&*raw)
     }
 }
@@ -199,9 +196,9 @@ where
 impl<'a, A> Iterator for FifoIter<'a, A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
-    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+    type Item = &'a A::Pointer;
 
     fn next(&mut self) -> Option<Self::Item> {
         unsafe {
@@ -224,14 +221,14 @@ where
 unsafe impl<A> Send for Fifo<A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
 unsafe impl<A> Sync for Fifo<A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
@@ -242,42 +239,38 @@ unsafe impl Sync for FifoLink {}
 unsafe impl<'a, A> Send for FifoIter<'a, A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
 unsafe impl<'a, A> Sync for FifoIter<'a, A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
 impl<A> EvictionPolicy for Fifo<A>
 where
     A: Adapter<Link = FifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     type Adapter = A;
-
     type Config = FifoConfig;
 
     fn new(config: Self::Config) -> Self {
         Self::new(config)
     }
 
-    fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         self.insert(ptr)
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
-    ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         self.remove(ptr)
     }
 
-    fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+    fn access(&mut self, ptr: &A::Pointer) {
         self.access(ptr)
     }
 
@@ -285,7 +278,7 @@ where
         self.len()
     }
 
-    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
+    fn iter(&self) -> impl Iterator<Item = &'_ A::Pointer> {
         self.iter()
     }
 }
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index f10dbb5f..6b37c490 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -30,7 +30,7 @@ use crate::{
     collections::dlist::{DList, DListIter},
     core::{
         adapter::{Adapter, KeyAdapter, Link},
-        pointer::PointerOps,
+        pointer::Pointer,
     },
     intrusive_adapter,
 };
@@ -127,7 +127,7 @@ intrusive_adapter! { LfuLinkMainDListAdapter = NonNull<LfuLink>: LfuLink { link_
 pub struct Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     /// tiny lru list
     lru_tiny: DList<LfuLinkTinyDListAdapter>,
@@ -159,7 +159,7 @@ where
 impl<A> Drop for Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     fn drop(&mut self) {
         let mut to_remove = vec![];
@@ -175,7 +175,7 @@ where
 impl<A> Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     pub fn new(config: LfuConfig) -> Self {
         let mut res = Self {
@@ -199,9 +199,9 @@ where
         res
     }
 
-    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let item = A::Pointer::into_raw(ptr);
             let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
 
             assert!(!link.as_ref().is_linked());
@@ -229,12 +229,9 @@ where
         }
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<A::PointerOps as PointerOps>::Pointer,
-    ) -> <A::PointerOps as PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let item = A::Pointer::as_ptr(ptr);
             let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
 
             assert!(link.as_ref().is_linked());
@@ -243,13 +240,13 @@ where
 
             self.len -= 1;
 
-            self.adapter.pointer_ops().from_raw(item)
+            A::Pointer::from_raw(item)
         }
     }
 
-    fn access(&mut self, ptr: &<A::PointerOps as PointerOps>::Pointer) {
+    fn access(&mut self, ptr: &A::Pointer) {
         unsafe {
-            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let item = A::Pointer::as_ptr(ptr);
             let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
 
             assert!(link.as_ref().is_linked());
@@ -414,34 +411,34 @@ where
 pub struct LfuIter<'a, A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     lfu: &'a Lfu<A>,
     iter_tiny: DListIter<'a, LfuLinkTinyDListAdapter>,
     iter_main: DListIter<'a, LfuLinkMainDListAdapter>,
 
-    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+    ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
 
 impl<'a, A> LfuIter<'a, A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     unsafe fn update_ptr(&mut self, link: NonNull<LfuLink>) {
         std::mem::forget(self.ptr.take());
 
         let item = self.lfu.adapter.link2item(link.as_ptr());
-        let ptr = self.lfu.adapter.pointer_ops().from_raw(item);
+        let ptr = A::Pointer::from_raw(item);
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
-    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+    unsafe fn ptr(&self) -> Option<&'a <A as Adapter>::Pointer> {
         if self.ptr.is_none() {
             return None;
         }
         let ptr = self.ptr.as_ref().unwrap();
-        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        let raw = ptr as *const <A as Adapter>::Pointer;
         Some(&*raw)
     }
 }
@@ -449,9 +446,9 @@ where
 impl<'a, A> Iterator for LfuIter<'a, A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
-    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+    type Item = &'a A::Pointer;
 
     fn next(&mut self) -> Option<Self::Item> {
         unsafe {
@@ -496,13 +493,13 @@ where
 unsafe impl<A> Send for Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 unsafe impl<A> Sync for Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
@@ -513,41 +510,37 @@ unsafe impl<'a, A> Send for LfuIter<'a, A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
 
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 unsafe impl<'a, A> Sync for LfuIter<'a, A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
 impl<A> EvictionPolicy for Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     type Adapter = A;
-
     type Config = LfuConfig;
 
     fn new(config: Self::Config) -> Self {
         Self::new(config)
     }
 
-    fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         self.insert(ptr)
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
-    ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         self.remove(ptr)
     }
 
-    fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+    fn access(&mut self, ptr: &A::Pointer) {
         self.access(ptr)
     }
 
@@ -555,7 +548,7 @@ where
         self.len()
     }
 
-    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
+    fn iter(&self) -> impl Iterator<Item = &'_ A::Pointer> {
         self.iter()
     }
 }
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 1e53ebab..dec035c1 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -32,7 +32,7 @@ use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, Link},
-        pointer::PointerOps,
+        pointer::Pointer,
     },
     intrusive_adapter,
 };
@@ -69,7 +69,7 @@ intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DLis
 pub struct Lru<A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     /// lru list
     lru: DList<LruLinkAdapter>,
@@ -90,7 +90,7 @@ where
 impl<A> Drop for Lru<A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     fn drop(&mut self) {
         let mut to_remove = vec![];
@@ -106,7 +106,7 @@ where
 impl<A> Lru<A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     fn new(config: LruConfig) -> Self {
         Self {
@@ -124,9 +124,9 @@ where
         }
     }
 
-    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let item = A::Pointer::into_raw(ptr);
             let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
 
             assert!(!link.as_ref().is_linked());
@@ -139,12 +139,9 @@ where
         }
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<A::PointerOps as PointerOps>::Pointer,
-    ) -> <A::PointerOps as PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let item = A::Pointer::as_ptr(ptr);
             let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
 
             assert!(link.as_ref().is_linked());
@@ -161,13 +158,13 @@ where
 
             self.len -= 1;
 
-            self.adapter.pointer_ops().from_raw(item)
+            A::Pointer::from_raw(item)
         }
     }
 
-    fn access(&mut self, ptr: &<A::PointerOps as PointerOps>::Pointer) {
+    fn access(&mut self, ptr: &A::Pointer) {
         unsafe {
-            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let item = A::Pointer::as_ptr(ptr);
             let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
 
             assert!(link.as_ref().is_linked());
@@ -290,33 +287,33 @@ where
 pub struct LruIter<'a, A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     lru: &'a Lru<A>,
     iter: DListIter<'a, LruLinkAdapter>,
 
-    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+    ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
 
 impl<'a, A> LruIter<'a, A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     unsafe fn update_ptr(&mut self, link: NonNull<LruLink>) {
         std::mem::forget(self.ptr.take());
 
         let item = self.lru.adapter.link2item(link.as_ptr());
-        let ptr = self.lru.adapter.pointer_ops().from_raw(item);
+        let ptr = A::Pointer::from_raw(item);
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
-    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+    unsafe fn ptr(&self) -> Option<&'a <A as Adapter>::Pointer> {
         if self.ptr.is_none() {
             return None;
         }
         let ptr = self.ptr.as_ref().unwrap();
-        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        let raw = ptr as *const <A as Adapter>::Pointer;
         Some(&*raw)
     }
 }
@@ -324,9 +321,9 @@ where
 impl<'a, A> Iterator for LruIter<'a, A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
-    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+    type Item = &'a A::Pointer;
 
     fn next(&mut self) -> Option<Self::Item> {
         unsafe {
@@ -349,14 +346,14 @@ where
 unsafe impl<A> Send for Lru<A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
 unsafe impl<A> Sync for Lru<A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
@@ -367,42 +364,38 @@ unsafe impl Sync for LruLink {}
 unsafe impl<'a, A> Send for LruIter<'a, A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
 unsafe impl<'a, A> Sync for LruIter<'a, A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
 }
 
 impl<A> EvictionPolicy for Lru<A>
 where
     A: Adapter<Link = LruLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <A as Adapter>::Pointer: Clone,
 {
     type Adapter = A;
-
     type Config = LruConfig;
 
     fn new(config: Self::Config) -> Self {
         Self::new(config)
     }
 
-    fn insert(&mut self, ptr: <<A>::PointerOps as PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         self.insert(ptr)
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<<A>::PointerOps as PointerOps>::Pointer,
-    ) -> <<A>::PointerOps as PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         self.remove(ptr)
     }
 
-    fn access(&mut self, ptr: &<<A>::PointerOps as PointerOps>::Pointer) {
+    fn access(&mut self, ptr: &A::Pointer) {
         self.access(ptr)
     }
 
@@ -410,7 +403,7 @@ where
         self.len()
     }
 
-    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
+    fn iter(&self) -> impl Iterator<Item = &'_ A::Pointer> {
         self.iter()
     }
 }
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index aeb966ba..c9d28c7b 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -12,8 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::core::{adapter::Adapter, pointer::PointerOps};
-
+use crate::core::adapter::Adapter;
 use std::fmt::Debug;
 
 pub trait Config = Send + Sync + 'static + Debug + Clone;
@@ -24,14 +23,14 @@ pub trait EvictionPolicy: Send + Sync + 'static {
 
     fn new(config: Self::Config) -> Self;
 
-    fn insert(&mut self, ptr: <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer);
+    fn insert(&mut self, ptr: <Self::Adapter as Adapter>::Pointer);
 
     fn remove(
         &mut self,
-        ptr: &<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer,
-    ) -> <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer;
+        ptr: &<Self::Adapter as Adapter>::Pointer,
+    ) -> <Self::Adapter as Adapter>::Pointer;
 
-    fn access(&mut self, ptr: &<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer);
+    fn access(&mut self, ptr: &<Self::Adapter as Adapter>::Pointer);
 
     fn len(&self) -> usize;
 
@@ -39,28 +38,26 @@ pub trait EvictionPolicy: Send + Sync + 'static {
         self.len() == 0
     }
 
-    fn iter(
-        &self,
-    ) -> impl Iterator<Item = &'_ <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer> + '_;
+    fn iter(&self) -> impl Iterator<Item = &'_ <Self::Adapter as Adapter>::Pointer> + '_;
 }
 
 pub trait EvictionPolicyExt: EvictionPolicy {
-    fn push(&mut self, ptr: <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer);
+    fn push(&mut self, ptr: <Self::Adapter as Adapter>::Pointer);
 
-    fn pop(&mut self) -> Option<<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer>;
+    fn pop(&mut self) -> Option<<Self::Adapter as Adapter>::Pointer>;
 
-    fn peek(&self) -> Option<&<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer>;
+    fn peek(&self) -> Option<&<Self::Adapter as Adapter>::Pointer>;
 }
 
 impl<E: EvictionPolicy> EvictionPolicyExt for E
 where
-    <<E::Adapter as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    <E::Adapter as Adapter>::Pointer: Clone,
 {
-    fn push(&mut self, ptr: <<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer) {
+    fn push(&mut self, ptr: <Self::Adapter as Adapter>::Pointer) {
         self.insert(ptr)
     }
 
-    fn pop(&mut self) -> Option<<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer> {
+    fn pop(&mut self) -> Option<<E::Adapter as Adapter>::Pointer> {
         let ptr = {
             let mut iter = self.iter();
             let ptr = iter.next();
@@ -69,7 +66,7 @@ where
         ptr.map(|ptr| self.remove(&ptr))
     }
 
-    fn peek(&self) -> Option<&<<Self::Adapter as Adapter>::PointerOps as PointerOps>::Pointer> {
+    fn peek(&self) -> Option<&<Self::Adapter as Adapter>::Pointer> {
         self.iter().next()
     }
 }
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index 2f5445d3..05722596 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -34,7 +34,7 @@ use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, Link, PriorityAdapter},
-        pointer::PointerOps,
+        pointer::Pointer,
     },
     intrusive_adapter,
 };
@@ -106,7 +106,7 @@ intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: Segm
 pub struct SegmentedFifo<A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
     // Note: All queue share the same dlist link.
     segments: Vec<DList<SegmentedFifoLinkAdapter>>,
@@ -122,7 +122,7 @@ where
 impl<A> Drop for SegmentedFifo<A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
     fn drop(&mut self) {
         let mut to_remove = vec![];
@@ -138,7 +138,7 @@ where
 impl<A> SegmentedFifo<A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
     pub fn new(config: SegmentedFifoConfig) -> Self {
         let segments = (0..config.segment_ratios.len())
@@ -158,9 +158,9 @@ where
         }
     }
 
-    fn insert(&mut self, ptr: <A::PointerOps as PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = self.adapter.pointer_ops().into_raw(ptr);
+            let item = A::Pointer::into_raw(ptr);
             let mut link =
                 NonNull::new_unchecked(self.adapter.item2link(item) as *mut SegmentedFifoLink);
 
@@ -177,12 +177,9 @@ where
         }
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<A::PointerOps as PointerOps>::Pointer,
-    ) -> <A::PointerOps as PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = self.adapter.pointer_ops().as_ptr(ptr);
+            let item = A::Pointer::as_ptr(ptr);
             let link =
                 NonNull::new_unchecked(self.adapter.item2link(item) as *mut SegmentedFifoLink);
 
@@ -198,11 +195,11 @@ where
 
             self.len -= 1;
 
-            self.adapter.pointer_ops().from_raw(item)
+            A::Pointer::from_raw(item)
         }
     }
 
-    fn access(&mut self, _ptr: &<A::PointerOps as PointerOps>::Pointer) {}
+    fn access(&mut self, _ptr: &A::Pointer) {}
 
     fn len(&self) -> usize {
         self.len
@@ -249,34 +246,34 @@ where
 pub struct SegmentedFifoIter<'a, A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
     sfifo: &'a SegmentedFifo<A>,
     iter_segments: Vec<DListIter<'a, SegmentedFifoLinkAdapter>>,
     segment: usize,
 
-    ptr: ManuallyDrop<Option<<<A as Adapter>::PointerOps as PointerOps>::Pointer>>,
+    ptr: ManuallyDrop<Option<A::Pointer>>,
 }
 
 impl<'a, A> SegmentedFifoIter<'a, A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
     unsafe fn update_ptr(&mut self, link: NonNull<SegmentedFifoLink>) {
         std::mem::forget(self.ptr.take());
 
         let item = self.sfifo.adapter.link2item(link.as_ptr());
-        let ptr = self.sfifo.adapter.pointer_ops().from_raw(item);
+        let ptr = A::Pointer::from_raw(item);
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
-    unsafe fn ptr(&self) -> Option<&'a <<A as Adapter>::PointerOps as PointerOps>::Pointer> {
+    unsafe fn ptr(&self) -> Option<&'a A::Pointer> {
         if self.ptr.is_none() {
             return None;
         }
         let ptr = self.ptr.as_ref().unwrap();
-        let raw = ptr as *const <<A as Adapter>::PointerOps as PointerOps>::Pointer;
+        let raw = ptr as *const A::Pointer;
         Some(&*raw)
     }
 }
@@ -284,9 +281,9 @@ where
 impl<'a, A> Iterator for SegmentedFifoIter<'a, A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
-    type Item = &'a <A::PointerOps as PointerOps>::Pointer;
+    type Item = &'a A::Pointer;
 
     fn next(&mut self) -> Option<Self::Item> {
         unsafe {
@@ -317,14 +314,14 @@ where
 unsafe impl<A> Send for SegmentedFifo<A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
 }
 
 unsafe impl<A> Sync for SegmentedFifo<A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
 }
 
@@ -335,42 +332,38 @@ unsafe impl Sync for SegmentedFifoLink {}
 unsafe impl<'a, A> Send for SegmentedFifoIter<'a, A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
 }
 
 unsafe impl<'a, A> Sync for SegmentedFifoIter<'a, A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
 }
 
 impl<A> EvictionPolicy for SegmentedFifo<A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
-    <<A as Adapter>::PointerOps as PointerOps>::Pointer: Clone,
+    A::Pointer: Clone,
 {
     type Adapter = A;
-
     type Config = SegmentedFifoConfig;
 
     fn new(config: Self::Config) -> Self {
         Self::new(config)
     }
 
-    fn insert(&mut self, ptr: <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+    fn insert(&mut self, ptr: A::Pointer) {
         self.insert(ptr)
     }
 
-    fn remove(
-        &mut self,
-        ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer,
-    ) -> <<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer {
+    fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         self.remove(ptr)
     }
 
-    fn access(&mut self, ptr: &<<A>::PointerOps as crate::core::pointer::PointerOps>::Pointer) {
+    fn access(&mut self, ptr: &A::Pointer) {
         self.access(ptr)
     }
 
@@ -378,7 +371,7 @@ where
         self.len()
     }
 
-    fn iter(&self) -> impl Iterator<Item = &'_ <A::PointerOps as PointerOps>::Pointer> {
+    fn iter(&self) -> impl Iterator<Item = &'_ A::Pointer> {
         self.iter()
     }
 }

From 8f7a7aecedb99d3cbb76b48d382996a7d5c5e7b4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 3 Aug 2023 17:26:12 +0800
Subject: [PATCH 076/261] chore: add codecov badge in README (#101)

Although it's bad for now...

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0b616093..e89bb373 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # foyer
 
-[![CI (main)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml) [![License Checker](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml)
+[![CI (main)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml) [![License Checker](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml) [![codecov](https://codecov.io/github/MrCroxx/foyer/branch/main/graph/badge.svg?token=YO33YQCB70)](https://codecov.io/github/MrCroxx/foyer)
 
 *foyer* aims to be a user-friendly hybrid cache lib in Rust. 
 

From 2a37f4a4f6aeede5c9ba88eaeec133e52576547e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 4 Aug 2023 13:31:32 +0800
Subject: [PATCH 077/261] chore: update ci to fix codecov (#102)

---
 .github/template/template.yml      | 5 +++--
 .github/workflows/main.yml         | 5 +++--
 .github/workflows/pull-request.yml | 5 +++--
 .gitignore                         | 1 +
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 100dbf80..1e794cdd 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -72,11 +72,12 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common --lib -- rated_random::tests --nocapture --ignored
           cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
       - name: Run rust test with coverage
         run: |
-          cargo llvm-cov nextest --lcov --output-path lcov.info
+          cargo llvm-cov --no-report nextest
+          cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
   deadlock:
     name: run with single worker thread and deadlock detection
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 393b080a..0e755fe0 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -79,11 +79,12 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common --lib -- rated_random::tests --nocapture --ignored
           cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
       - name: Run rust test with coverage
         run: |
-          cargo llvm-cov nextest --lcov --output-path lcov.info
+          cargo llvm-cov --no-report nextest
+          cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
   deadlock:
     name: run with single worker thread and deadlock detection
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index a01681d8..7e380a48 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -78,11 +78,12 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-storage --lib -- admission::rated_random::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common --lib -- rated_random::tests --nocapture --ignored
           cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
       - name: Run rust test with coverage
         run: |
-          cargo llvm-cov nextest --lcov --output-path lcov.info
+          cargo llvm-cov --no-report nextest
+          cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
   deadlock:
     name: run with single worker thread and deadlock detection
diff --git a/.gitignore b/.gitignore
index 22d3441b..bfefebf6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@
 /target
 
 Cargo.lock
+lcov.info
\ No newline at end of file

From 552a95fc02d2a50518697edb998989c33d588af6 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 4 Aug 2023 14:57:32 +0800
Subject: [PATCH 078/261] chore: ignore some code cov to make it more accurate
 (#103)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/code.rs | 18 ++++++++++++++++++
 foyer-common/src/lib.rs  |  1 +
 2 files changed, 19 insertions(+)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 60aaaf26..3b7e4811 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -29,18 +29,22 @@ pub trait Key:
     + Clone
     + std::fmt::Debug
 {
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Key` if storage is used.")
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
@@ -48,18 +52,22 @@ pub trait Key:
 
 #[allow(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Value` if storage is used.")
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
@@ -79,14 +87,17 @@ macro_rules! impl_key {
         paste! {
             $(
                 impl Key for $type {
+                    #[cfg_attr(coverage_nightly, no_coverage)]
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
                     }
 
+                    #[cfg_attr(coverage_nightly, no_coverage)]
                     fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
+                    #[cfg_attr(coverage_nightly, no_coverage)]
                     fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
@@ -101,14 +112,17 @@ macro_rules! impl_value {
         paste! {
             $(
                 impl Value for $type {
+                    #[cfg_attr(coverage_nightly, no_coverage)]
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
                     }
 
+                    #[cfg_attr(coverage_nightly, no_coverage)]
                     fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
+                    #[cfg_attr(coverage_nightly, no_coverage)]
                     fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
@@ -122,18 +136,22 @@ for_all_primitives! { impl_key }
 for_all_primitives! { impl_value }
 
 impl Value for Vec<u8> {
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn weight(&self) -> usize {
         self.len()
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn serialized_len(&self) -> usize {
         self.len()
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn write(&self, mut buf: &mut [u8]) {
         buf.put_slice(self);
     }
 
+    #[cfg_attr(coverage_nightly, no_coverage)]
     fn read(buf: &[u8]) -> Self {
         buf.to_vec()
     }
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 1aa9d2cc..e12637d6 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -13,6 +13,7 @@
 //  limitations under the License.
 
 #![feature(trait_alias)]
+#![cfg_attr(coverage_nightly, feature(no_coverage))]
 
 pub mod batch;
 pub mod bits;

From 1c51add9b332d7b215f5a4db21659580840bff4e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 8 Aug 2023 20:09:22 +0800
Subject: [PATCH 079/261] Create dependabot.yml (#104)

---
 .github/dependabot.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 .github/dependabot.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000..2b1d557c
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "cargo" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+    rebase-strategy: "disabled"

From 36eeac9ca2aa27d208065fab062ccbe57b63b54f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 8 Aug 2023 21:14:56 +0800
Subject: [PATCH 080/261] chore(deps): update memoffset requirement from 0.8 to
 0.9 (#106)

Updates the requirements on [memoffset](https://github.com/Gilnaa/memoffset) to permit the latest version.
- [Changelog](https://github.com/Gilnaa/memoffset/blob/master/CHANGELOG.md)
- [Commits](https://github.com/Gilnaa/memoffset/compare/v0.8.0...v0.9.0)

---
updated-dependencies:
- dependency-name: memoffset
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 foyer-intrusive/Cargo.toml | 2 +-
 foyer-memory/Cargo.toml    | 2 +-
 foyer-storage/Cargo.toml   | 2 +-
 foyer/Cargo.toml           | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 2667713f..13720df3 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -14,7 +14,7 @@ cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 itertools = "0.10.5"
-memoffset = "0.8"
+memoffset = "0.9"
 parking_lot = "0.12"
 paste = "1.0"
 thiserror = "1"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index ce5bfd32..918a04e6 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -19,7 +19,7 @@ foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
-memoffset = "0.8"
+memoffset = "0.9"
 nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 1e02dd60..a13ed3de 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -21,7 +21,7 @@ foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
-memoffset = "0.8"
+memoffset = "0.9"
 nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 1e937b6d..670541e0 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -19,7 +19,7 @@ foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.10.5"
 libc = "0.2"
-memoffset = "0.8"
+memoffset = "0.9"
 nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"

From 1629daff63c07ea95b8795919136109e38dffc09 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 9 Aug 2023 12:09:39 +0800
Subject: [PATCH 081/261] chore(ci): use cargo-binstall to install tools (#107)

* chore(ci): use cargo-binstall to install tools

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      | 8 ++++++--
 .github/workflows/main.yml         | 8 ++++++--
 .github/workflows/pull-request.yml | 8 ++++++--
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 1e794cdd..b735b4b3 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -5,7 +5,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230703
+  CACHE_KEY_SUFFIX: 20230809
 
 jobs:
   misc-check:
@@ -47,10 +47,14 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Install cargo-binstall
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
       - name: Install cargo tools
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          cargo install cargo-sort cargo-hakari
+          cargo binstall -y cargo-sort cargo-hakari
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0e755fe0..9ca82a89 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230703
+  CACHE_KEY_SUFFIX: 20230809
 jobs:
   misc-check:
     name: misc check
@@ -54,10 +54,14 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Install cargo-binstall
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
       - name: Install cargo tools
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          cargo install cargo-sort cargo-hakari
+          cargo binstall -y cargo-sort cargo-hakari
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 7e380a48..cef1cc6e 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -12,7 +12,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230703
+  CACHE_KEY_SUFFIX: 20230809
 jobs:
   misc-check:
     name: misc check
@@ -53,10 +53,14 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Install cargo-binstall
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
       - name: Install cargo tools
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          cargo install cargo-sort cargo-hakari
+          cargo binstall -y cargo-sort cargo-hakari
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c

From b9497943d397f904fb719f80b908cd17ff019779 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 9 Aug 2023 04:14:13 +0000
Subject: [PATCH 082/261] chore(deps): update itertools requirement from 0.10.5
 to 0.11.0 (#105)

* chore(deps): update itertools requirement from 0.10.5 to 0.11.0

Updates the requirements on [itertools](https://github.com/rust-itertools/itertools) to permit the latest version.
- [Changelog](https://github.com/rust-itertools/itertools/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rust-itertools/itertools/compare/v0.10.5...v0.11.0)

---
updated-dependencies:
- dependency-name: itertools
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

* make hakiri happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: MrCroxx <mrcroxx@outlook.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml        | 2 +-
 foyer-memory/Cargo.toml        | 2 +-
 foyer-storage-bench/Cargo.toml | 2 +-
 foyer-storage/Cargo.toml       | 2 +-
 foyer/Cargo.toml               | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index cb8eb94c..1582fe6f 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -24,4 +24,4 @@ tokio = { version = "1", features = [
 tracing = "0.1"
 
 [dev-dependencies]
-itertools = "0.10.5"
+itertools = "0.11.0"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 918a04e6..53bfd024 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -17,7 +17,7 @@ foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
-itertools = "0.10.5"
+itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
 nix = { version = "0.26", features = ["fs", "mman"] }
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 0a3158ff..7c17d355 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -16,7 +16,7 @@ foyer-storage = { path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
-itertools = "0.10.5"
+itertools = "0.11.0"
 libc = "0.2"
 nix = { version = "0.26", features = ["fs", "mman"] }
 parking_lot = "0.12"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index a13ed3de..5a26d9fa 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -19,7 +19,7 @@ foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
-itertools = "0.10.5"
+itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
 nix = { version = "0.26", features = ["fs", "mman"] }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 670541e0..adb4b738 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -17,7 +17,7 @@ foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-storage = { path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
-itertools = "0.10.5"
+itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
 nix = { version = "0.26", features = ["fs", "mman"] }

From b8af0b66f2aa1ed9f965b3fb2cc020047bb80490 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 9 Aug 2023 12:41:32 +0800
Subject: [PATCH 083/261] chore: update makefile deps (#108)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                | 2 +-
 scripts/install-deps.sh | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100755 scripts/install-deps.sh

diff --git a/Makefile b/Makefile
index f1ca3065..8928a890 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ SHELL := /bin/bash
 .PHONY: proto check test deps
 
 deps:
-	cargo install cargo-hakari cargo-sort
+	./scripts/install-deps.sh
 
 check:
 	cargo hakari generate
diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh
new file mode 100755
index 00000000..357896a6
--- /dev/null
+++ b/scripts/install-deps.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [ -z "$(which cargo-binstall)" ]; then
+    curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
+fi
+
+cargo binstall -y cargo-hakari cargo-sort
\ No newline at end of file

From c826f70ec1b5c9cad889ee4b4eb9ba8f25837112 Mon Sep 17 00:00:00 2001
From: xxchan <xxchan22f@gmail.com>
Date: Tue, 22 Aug 2023 11:57:06 +0200
Subject: [PATCH 084/261] fix: build on latest nightly (#109)

---
 .github/template/template.yml      | 4 ++--
 .github/workflows/main.yml         | 4 ++--
 .github/workflows/pull-request.yml | 4 ++--
 foyer-storage/src/lib.rs           | 1 -
 rust-toolchain                     | 2 +-
 5 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index b735b4b3..9771e45e 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -3,9 +3,9 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-31
+  RUST_TOOLCHAIN: nightly-2023-08-21
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230809
+  CACHE_KEY_SUFFIX: 20230822
 
 jobs:
   misc-check:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9ca82a89..4f13f835 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,9 +11,9 @@ on:
     branches: [main]
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-31
+  RUST_TOOLCHAIN: nightly-2023-08-21
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230809
+  CACHE_KEY_SUFFIX: 20230822
 jobs:
   misc-check:
     name: misc check
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index cef1cc6e..9ab0fc88 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -10,9 +10,9 @@ on:
   pull_request:
     branches: [main]
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-31
+  RUST_TOOLCHAIN: nightly-2023-08-21
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230809
+  CACHE_KEY_SUFFIX: 20230822
 jobs:
   misc-check:
     name: misc check
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index b19927ce..004cde4d 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -17,7 +17,6 @@
 #![feature(trait_alias)]
 #![feature(get_mut_unchecked)]
 #![feature(let_chains)]
-#![feature(provide_any)]
 #![feature(error_generic_member_access)]
 #![allow(clippy::type_complexity)]
 
diff --git a/rust-toolchain b/rust-toolchain
index b0adca9d..ece8aeba 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-05-31"
\ No newline at end of file
+channel = "nightly-2023-08-21"
\ No newline at end of file

From 0f4f8081e769c94ea477cbf2fea317ef524055d8 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 23 Aug 2023 11:18:06 +0800
Subject: [PATCH 085/261] chore: update readme (#110)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index e89bb373..67397938 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,6 @@ Currently, *foyer* only finished few features, and is still under heavy developm
 
 - [ ] Better intrusive index collectios for memory cache.
 - [ ] Hybrid memory and disk cache.
-- [ ] Reinsertion for disk cache.
 - [ ] Raw device and single file device support.
 - [ ] More detailed metrics or statistics.
 

From c6e3573cb289ebb0c8edc6523b04e107c080220d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 23 Aug 2023 11:40:11 +0800
Subject: [PATCH 086/261] chore: fix workspace (#111)

* fix: workspace

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update lock file

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml                      | 2 +-
 foyer-workspace-hack/Cargo.toml | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index ec34e159..54508932 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
 [workspace]
-
+resolver = "2"
 members = [
     "foyer",
     "foyer-common",
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 61591aa3..3f450345 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -20,9 +20,6 @@ futures-channel = { version = "0.3", features = ["sink"] }
 futures-core = { version = "0.3" }
 futures-sink = { version = "0.3" }
 itertools = { version = "0.10" }
-libc = { version = "0.2", features = ["extra_traits"] }
-memchr = { version = "2" }
-miniz_oxide = { version = "0.7", default-features = false, features = ["with-alloc"] }
 parking_lot = { version = "0.12", features = ["deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }

From 86e2062cf229b9782b339fcdd4ff48b42d74231d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 28 Aug 2023 14:22:54 +0800
Subject: [PATCH 087/261] fix: fix insert duration (#112)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/store.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 3a78141c..d82943a3 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -610,12 +610,12 @@ where
     ) -> Result<bool> {
         debug_assert!(!writer.is_inserted);
 
-        let now = Instant::now();
-
         if !writer.judge() {
             return Ok(false);
         }
 
+        let now = Instant::now();
+
         writer.is_inserted = true;
         let key = &writer.key;
 

From 99b21dfd0759bc7b9b04c46e952a7283c36b2672 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 29 Aug 2023 14:22:42 +0800
Subject: [PATCH 088/261] chore: downgrade rust toolchain (#114)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      | 4 ++--
 .github/workflows/main.yml         | 4 ++--
 .github/workflows/pull-request.yml | 4 ++--
 foyer-storage/src/device/error.rs  | 5 ++---
 foyer-storage/src/error.rs         | 5 ++---
 rust-toolchain                     | 2 +-
 6 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 9771e45e..eefdb765 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -3,9 +3,9 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-08-21
+  RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230822
+  CACHE_KEY_SUFFIX: 20230829
 
 jobs:
   misc-check:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4f13f835..342081c7 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,9 +11,9 @@ on:
     branches: [main]
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2023-08-21
+  RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230822
+  CACHE_KEY_SUFFIX: 20230829
 jobs:
   misc-check:
     name: misc check
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 9ab0fc88..c81280fc 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -10,9 +10,9 @@ on:
   pull_request:
     branches: [main]
 env:
-  RUST_TOOLCHAIN: nightly-2023-08-21
+  RUST_TOOLCHAIN: nightly-2023-05-31
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230822
+  CACHE_KEY_SUFFIX: 20230829
 jobs:
   misc-check:
     name: misc check
diff --git a/foyer-storage/src/device/error.rs b/foyer-storage/src/device/error.rs
index bdbbc004..5b595cda 100644
--- a/foyer-storage/src/device/error.rs
+++ b/foyer-storage/src/device/error.rs
@@ -12,8 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::backtrace::Backtrace;
-
 #[derive(thiserror::Error, Debug)]
 #[error("{0}")]
 pub struct DeviceError(Box<DeviceErrorInner>);
@@ -22,7 +20,8 @@ pub struct DeviceError(Box<DeviceErrorInner>);
 #[error("{source}")]
 struct DeviceErrorInner {
     source: DeviceErrorKind,
-    backtrace: Backtrace,
+    // https://github.com/dtolnay/thiserror/issues/204
+    // backtrace: Backtrace,
 }
 
 #[derive(thiserror::Error, Debug)]
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index d4dde62a..a48ac3b3 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -12,8 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::backtrace::Backtrace;
-
 use crate::device::error::DeviceError;
 
 #[derive(thiserror::Error, Debug)]
@@ -24,7 +22,8 @@ pub struct Error(Box<ErrorInner>);
 #[error("{source}")]
 struct ErrorInner {
     source: ErrorKind,
-    backtrace: Backtrace,
+    // https://github.com/dtolnay/thiserror/issues/204
+    // backtrace: Backtrace,
 }
 
 #[derive(thiserror::Error, Debug)]
diff --git a/rust-toolchain b/rust-toolchain
index ece8aeba..b0adca9d 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-08-21"
\ No newline at end of file
+channel = "nightly-2023-05-31"
\ No newline at end of file

From cbf9fc6badc625085c2a1271b26592e41cad4fd7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 30 Aug 2023 04:33:09 +0000
Subject: [PATCH 089/261] chore(deps): update nix requirement from 0.26 to 0.27
 (#113)

* chore(deps): update nix requirement from 0.26 to 0.27

Updates the requirements on [nix](https://github.com/nix-rust/nix) to permit the latest version.
- [Changelog](https://github.com/nix-rust/nix/blob/master/CHANGELOG.md)
- [Commits](https://github.com/nix-rust/nix/compare/v0.26.0...v0.27.1)

---
updated-dependencies:
- dependency-name: nix
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix updates

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: MrCroxx <mrcroxx@outlook.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/Cargo.toml         | 2 +-
 foyer-storage-bench/Cargo.toml  | 2 +-
 foyer-storage/Cargo.toml        | 2 +-
 foyer-storage/src/device/fs.rs  | 4 +++-
 foyer-workspace-hack/Cargo.toml | 1 +
 foyer/Cargo.toml                | 2 +-
 6 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 53bfd024..cb8ed171 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -20,7 +20,7 @@ futures = "0.3"
 itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
-nix = { version = "0.26", features = ["fs", "mman"] }
+nix = { version = "0.27", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
 pin-project = "1"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 7c17d355..b8e83f4b 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -18,7 +18,7 @@ futures = "0.3"
 hdrhistogram = "7"
 itertools = "0.11.0"
 libc = "0.2"
-nix = { version = "0.26", features = ["fs", "mman"] }
+nix = { version = "0.27", features = ["fs", "mman"] }
 parking_lot = "0.12"
 rand = "0.8.5"
 rand_mt = "4.2.1"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 5a26d9fa..f5556f92 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -22,7 +22,7 @@ futures = "0.3"
 itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
-nix = { version = "0.26", features = ["fs", "mman"] }
+nix = { version = "0.27", features = ["fs", "mman", "uio"] }
 parking_lot = "0.12"
 paste = "1.0"
 pin-project = "1"
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 49333059..6f8743c4 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -14,7 +14,7 @@
 
 use std::{
     fs::{create_dir_all, File, OpenOptions},
-    os::fd::{AsRawFd, RawFd},
+    os::fd::{AsRawFd, BorrowedFd, RawFd},
     path::PathBuf,
     sync::Arc,
 };
@@ -96,6 +96,7 @@ impl Device for FsDevice {
         let fd = self.fd(region);
 
         let res = asyncify(move || {
+            let fd = unsafe { BorrowedFd::borrow_raw(fd) };
             let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[..len], offset as i64)?;
             Ok(res)
         })
@@ -117,6 +118,7 @@ impl Device for FsDevice {
         let fd = self.fd(region);
 
         let res = asyncify(move || {
+            let fd = unsafe { BorrowedFd::borrow_raw(fd) };
             let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[..len], offset as i64)?;
             Ok(res)
         })
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 3f450345..f7e7134c 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -20,6 +20,7 @@ futures-channel = { version = "0.3", features = ["sink"] }
 futures-core = { version = "0.3" }
 futures-sink = { version = "0.3" }
 itertools = { version = "0.10" }
+nix = { version = "0.27", features = ["fs", "mman", "uio"] }
 parking_lot = { version = "0.12", features = ["deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index adb4b738..0c1352d8 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -20,7 +20,7 @@ futures = "0.3"
 itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
-nix = { version = "0.26", features = ["fs", "mman"] }
+nix = { version = "0.27", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
 prometheus = "0.13"

From d78c8647c70e2bdb74baa75337017bf6554b5a44 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 1 Sep 2023 15:15:10 +0800
Subject: [PATCH 090/261] refactor: use rust-prometheus global registry (#115)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/lib.rs      |   6 +-
 foyer-storage-bench/src/main.rs |   4 +-
 foyer-storage/src/flusher.rs    |   8 +-
 foyer-storage/src/lib.rs        |   3 +-
 foyer-storage/src/metrics.rs    | 175 +++++++++++++++++++-------------
 foyer-storage/src/reclaimer.rs  |  11 +-
 foyer-storage/src/store.rs      |  55 ++++------
 7 files changed, 142 insertions(+), 120 deletions(-)

diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 717ac423..09b0b191 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -16,9 +16,9 @@
 #![feature(ptr_metadata)]
 #![feature(trait_alias)]
 #![feature(return_position_impl_trait_in_trait)]
-#![allow(clippy::new_without_default)]
-#![allow(clippy::wrong_self_convention)]
-#![allow(clippy::vtable_address_comparisons)]
+#![feature(lint_reasons)]
+#![expect(clippy::new_without_default)]
+#![cfg_attr(test, expect(clippy::vtable_address_comparisons))]
 
 pub use memoffset::offset_of;
 
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 8f5a1ac9..4880f65a 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -36,7 +36,7 @@ use foyer_storage::{
     admission::{rated_random::RatedRandomAdmissionPolicy, AdmissionPolicy},
     device::fs::FsDeviceConfig,
     reinsertion::{rated_random::RatedRandomReinsertionPolicy, ReinsertionPolicy},
-    store::{PrometheusConfig, StoreConfig},
+    store::StoreConfig,
     LfuFsStore,
 };
 use futures::future::join_all;
@@ -254,6 +254,7 @@ async fn main() {
     };
 
     let config = StoreConfig {
+        name: "".to_string(),
         eviction_config,
         device_config,
         admissions,
@@ -265,7 +266,6 @@ async fn main() {
         reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
         event_listeners: vec![],
-        prometheus_config: PrometheusConfig::default(),
         clean_region_threshold,
     };
 
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 7add1da3..a79c3a39 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -80,6 +80,8 @@ where
     }
 
     async fn handle(&self, region_id: RegionId) -> Result<()> {
+        let _timer = self.metrics.op_duration_flush.start_timer();
+
         tracing::info!("[flusher] receive flush task, region: {}", region_id);
 
         let region = self.region_manager.region(&region_id);
@@ -131,9 +133,11 @@ where
         tracing::info!("[flusher] finish flush task, region: {}", region_id);
 
         self.metrics
-            .bytes_flush
+            .op_bytes_flush
             .inc_by(region.device().region_size() as u64);
-        self.metrics.size.add(region.device().region_size() as i64);
+        self.metrics
+            .total_bytes
+            .add(region.device().region_size() as i64);
 
         Ok(())
     }
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 004cde4d..92172d98 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -18,7 +18,8 @@
 #![feature(get_mut_unchecked)]
 #![feature(let_chains)]
 #![feature(error_generic_member_access)]
-#![allow(clippy::type_complexity)]
+#![feature(lazy_cell)]
+#![feature(lint_reasons)]
 
 pub mod admission;
 pub mod device;
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index e1c83ae4..a4a853d5 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -12,98 +12,127 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::sync::LazyLock;
+
 use prometheus::{
-    register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
-    register_int_gauge_with_registry, Histogram, HistogramOpts, IntCounter, IntGauge, Opts,
-    Registry,
+    register_histogram_vec, register_int_counter_vec, register_int_gauge_vec, Histogram,
+    HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
 };
 
+/// Multiple foyer instance will share the same global metrics with different label `foyer` name.
+pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
+
 #[derive(Debug)]
-pub struct Metrics {
-    pub latency_insert_inserted: Histogram,
-    pub latency_insert_filtered: Histogram,
-    pub latency_insert_dropped: Histogram,
-    pub latency_lookup_hit: Histogram,
-    pub latency_lookup_miss: Histogram,
-    pub latency_remove: Histogram,
-
-    pub bytes_insert: IntCounter,
-    pub bytes_lookup: IntCounter,
-    pub bytes_flush: IntCounter,
-    pub bytes_reclaim: IntCounter,
-    pub bytes_reinsert: IntCounter,
-
-    pub size: IntGauge,
+pub struct GlobalMetrics {
+    op_duration: HistogramVec,
+    op_bytes: IntCounterVec,
+    total_bytes: IntGaugeVec,
 }
 
-impl Default for Metrics {
+impl Default for GlobalMetrics {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl Metrics {
+impl GlobalMetrics {
     pub fn new() -> Self {
-        Self::with_registry_namespace(Registry::default(), "")
-    }
+        let op_duration = register_histogram_vec!(
+            "foyer_storage_op_duration",
+            "foyer storage op duration",
+            &["foyer", "op", "extra"],
+            vec![0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
+        )
+        .unwrap();
 
-    pub fn with_namespace(namespace: impl ToString) -> Self {
-        Self::with_registry_namespace(Registry::default(), namespace)
+        let op_bytes = register_int_counter_vec!(
+            "foyer_storage_op_bytes",
+            "foyer storage op bytes",
+            &["foyer", "op", "extra"]
+        )
+        .unwrap();
+
+        let total_bytes =
+            register_int_gauge_vec!("total_bytes", "total bytes", &["foyer"]).unwrap();
+
+        Self {
+            op_duration,
+            op_bytes,
+            total_bytes,
+        }
     }
 
-    pub fn with_registry(registry: Registry) -> Self {
-        Self::with_registry_namespace(registry, "")
+    pub fn foyer(&self, name: &str) -> Metrics {
+        Metrics::new(self, name)
     }
+}
+
+#[derive(Debug)]
+pub struct Metrics {
+    pub op_duration_insert_inserted: Histogram,
+    pub op_duration_insert_filtered: Histogram,
+    pub op_duration_insert_dropped: Histogram,
+    pub op_duration_lookup_hit: Histogram,
+    pub op_duration_lookup_miss: Histogram,
+    pub op_duration_remove: Histogram,
+    pub op_duration_flush: Histogram,
+    pub op_duration_reclaim: Histogram,
+
+    pub op_bytes_insert: IntCounter,
+    pub op_bytes_lookup: IntCounter,
+    pub op_bytes_flush: IntCounter,
+    pub op_bytes_reclaim: IntCounter,
+    pub op_bytes_reinsert: IntCounter,
+
+    pub total_bytes: IntGauge,
+}
+
+impl Metrics {
+    pub fn new(global: &GlobalMetrics, foyer: &str) -> Self {
+        let op_duration_insert_inserted = global
+            .op_duration
+            .with_label_values(&[foyer, "insert", "inserted"]);
+        let op_duration_insert_filtered = global
+            .op_duration
+            .with_label_values(&[foyer, "insert", "filtered"]);
+        let op_duration_insert_dropped = global
+            .op_duration
+            .with_label_values(&[foyer, "insert", "dropped"]);
+        let op_duration_lookup_hit = global
+            .op_duration
+            .with_label_values(&[foyer, "lookup", "hit"]);
+        let op_duration_lookup_miss = global
+            .op_duration
+            .with_label_values(&[foyer, "lookup", "miss"]);
+        let op_duration_remove = global.op_duration.with_label_values(&[foyer, "remove", ""]);
+        let op_duration_flush = global.op_duration.with_label_values(&[foyer, "flush", ""]);
+        let op_duration_reclaim = global
+            .op_duration
+            .with_label_values(&[foyer, "reclaim", ""]);
+
+        let op_bytes_insert = global.op_bytes.with_label_values(&[foyer, "insert", ""]);
+        let op_bytes_lookup = global.op_bytes.with_label_values(&[foyer, "lookup", ""]);
+        let op_bytes_flush = global.op_bytes.with_label_values(&[foyer, "flush", ""]);
+        let op_bytes_reclaim = global.op_bytes.with_label_values(&[foyer, "reclaim", ""]);
+        let op_bytes_reinsert = global.op_bytes.with_label_values(&[foyer, "reinsert", ""]);
 
-    pub fn with_registry_namespace(registry: Registry, namespace: impl ToString) -> Self {
-        let latency = {
-            let opts = HistogramOpts::new("foyer_storage_latency", "foyer storage latency")
-                .namespace(namespace.to_string())
-                .buckets(vec![
-                    0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0,
-                ]);
-            register_histogram_vec_with_registry!(opts, &["op", "extra"], registry).unwrap()
-        };
-        let bytes = {
-            let opts = Opts::new("foyer_storage_bytes", "foyer storage bytes")
-                .namespace(namespace.to_string());
-            register_int_counter_vec_with_registry!(opts, &["op", "extra"], registry).unwrap()
-        };
-
-        let latency_insert_inserted = latency.with_label_values(&["insert", "inserted"]);
-        let latency_insert_filtered = latency.with_label_values(&["insert", "filtered"]);
-        let latency_insert_dropped = latency.with_label_values(&["insert", "dropped"]);
-        let latency_lookup_hit = latency.with_label_values(&["lookup", "hit"]);
-        let latency_lookup_miss = latency.with_label_values(&["lookup", "miss"]);
-        let latency_remove = latency.with_label_values(&["remove", ""]);
-
-        let bytes_insert = bytes.with_label_values(&["insert", ""]);
-        let bytes_lookup = bytes.with_label_values(&["lookup", ""]);
-        let bytes_flush = bytes.with_label_values(&["flush", ""]);
-        let bytes_reclaim = bytes.with_label_values(&["reclaim", ""]);
-        let bytes_reinsert = bytes.with_label_values(&["reinsert", ""]);
-
-        let size = {
-            let opts = Opts::new("foyer_storage_size", "foyer storage size")
-                .namespace(namespace.to_string());
-            register_int_gauge_with_registry!(opts, registry).unwrap()
-        };
+        let total_bytes = global.total_bytes.with_label_values(&[foyer]);
 
         Self {
-            latency_insert_inserted,
-            latency_insert_filtered,
-            latency_insert_dropped,
-            latency_lookup_hit,
-            latency_lookup_miss,
-            latency_remove,
-
-            bytes_insert,
-            bytes_lookup,
-            bytes_flush,
-            bytes_reclaim,
-            bytes_reinsert,
-
-            size,
+            op_duration_insert_inserted,
+            op_duration_insert_filtered,
+            op_duration_insert_dropped,
+            op_duration_lookup_hit,
+            op_duration_lookup_miss,
+            op_duration_remove,
+            op_duration_flush,
+            op_duration_reclaim,
+            op_bytes_insert,
+            op_bytes_lookup,
+            op_bytes_flush,
+            op_bytes_reclaim,
+            op_bytes_reinsert,
+            total_bytes,
         }
     }
 }
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index edb087f4..da399ce4 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -111,6 +111,9 @@ where
                 None => tokio::time::sleep(Duration::from_millis(100)).await,
             }
         };
+
+        let _timer = self.metrics.op_duration_reclaim.start_timer();
+
         let region = self.region_manager.region(&region_id);
 
         // step 1: drop indices
@@ -185,7 +188,7 @@ where
                         }
                     }
 
-                    metrics.bytes_reinsert.inc_by(weight as u64);
+                    metrics.op_bytes_reinsert.inc_by(weight as u64);
                 }
 
                 tracing::info!("[reclaimer] finish reinsertion, region: {}", region_id);
@@ -214,9 +217,11 @@ where
         tracing::info!("[reclaimer] finish reclaim task, region: {}", region_id);
 
         self.metrics
-            .bytes_reclaim
+            .op_bytes_reclaim
             .inc_by(region.device().region_size() as u64);
-        self.metrics.size.sub(region.device().region_size() as i64);
+        self.metrics
+            .total_bytes
+            .sub(region.device().region_size() as i64);
 
         Ok(())
     }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index d82943a3..547af2c1 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -36,7 +36,7 @@ use crate::{
     flusher::Flusher,
     indices::{Index, Indices},
     judge::Judges,
-    metrics::Metrics,
+    metrics::{Metrics, METRICS},
     reclaimer::Reclaimer,
     region::{AllocateResult, Region, RegionId},
     region_manager::{RegionEpItemAdapter, RegionManager},
@@ -51,12 +51,6 @@ const DEFAULT_BROADCAST_CAPACITY: usize = 4096;
 
 pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
 
-#[derive(Debug, Default)]
-pub struct PrometheusConfig {
-    pub registry: Option<prometheus::Registry>,
-    pub namespace: Option<String>,
-}
-
 pub struct StoreConfig<K, V, D, EP>
 where
     K: Key,
@@ -64,6 +58,11 @@ where
     D: Device,
     EP: EvictionPolicy,
 {
+    /// For distinguish different foyer metrics.
+    ///
+    /// Metrics of this foyer instance has label `foyer = {{ name }}`.
+    pub name: String,
+
     /// Evictino policy configurations.
     pub eviction_config: EP::Config,
 
@@ -101,9 +100,6 @@ where
 
     /// Event listsners.
     pub event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
-
-    /// Prometheus configuration.
-    pub prometheus_config: PrometheusConfig,
 }
 
 impl<K, V, D, EP> Debug for StoreConfig<K, V, D, EP>
@@ -191,18 +187,7 @@ where
             .map(|_| reclaimers_stop_tx.subscribe())
             .collect_vec();
 
-        let metrics = match (
-            config.prometheus_config.registry,
-            config.prometheus_config.namespace,
-        ) {
-            (Some(registry), Some(namespace)) => {
-                Metrics::with_registry_namespace(registry, namespace)
-            }
-            (Some(registry), None) => Metrics::with_registry(registry),
-            (None, Some(namespace)) => Metrics::with_namespace(namespace),
-            (None, None) => Metrics::new(),
-        };
-        let metrics = Arc::new(metrics);
+        let metrics = Arc::new(METRICS.foyer(&config.name));
 
         let store = Arc::new(Self {
             indices: indices.clone(),
@@ -417,7 +402,7 @@ where
             Some(index) => index,
             None => {
                 self.metrics
-                    .latency_lookup_miss
+                    .op_duration_lookup_miss
                     .observe(now.elapsed().as_secs_f64());
                 return Ok(None);
             }
@@ -435,12 +420,12 @@ where
                 // Remove index if the storage layer fails to lookup it (because of region version mismatch).
                 self.indices.remove(key);
                 self.metrics
-                    .latency_lookup_miss
+                    .op_duration_lookup_miss
                     .observe(now.elapsed().as_secs_f64());
                 return Ok(None);
             }
         };
-        self.metrics.bytes_lookup.inc_by(slice.len() as u64);
+        self.metrics.op_bytes_lookup.inc_by(slice.len() as u64);
 
         let res = match read_entry::<K, V>(slice.as_ref()) {
             Some((_key, value)) => Ok(Some(value)),
@@ -453,7 +438,7 @@ where
         slice.destroy().await;
 
         self.metrics
-            .latency_lookup_hit
+            .op_duration_lookup_hit
             .observe(now.elapsed().as_secs_f64());
 
         res
@@ -461,7 +446,7 @@ where
 
     #[tracing::instrument(skip(self))]
     pub async fn remove(&self, key: &K) -> Result<bool> {
-        let _timer = self.metrics.latency_remove.start_timer();
+        let _timer = self.metrics.op_duration_remove.start_timer();
 
         let res = self.indices.remove(key).is_some();
 
@@ -476,8 +461,6 @@ where
 
     #[tracing::instrument(skip(self))]
     pub async fn clear(&self) -> Result<()> {
-        let _timer = self.metrics.latency_remove.start_timer();
-
         self.indices.clear();
 
         for listener in self.event_listeners.iter() {
@@ -632,7 +615,7 @@ where
             );
         }
 
-        self.metrics.bytes_insert.inc_by(serialized_len as u64);
+        self.metrics.op_bytes_insert.inc_by(serialized_len as u64);
 
         let mut slice = loop {
             match self
@@ -677,7 +660,7 @@ where
 
         let duration = now.elapsed() + writer.duration;
         self.metrics
-            .latency_insert_inserted
+            .op_duration_insert_inserted
             .observe(duration.as_secs_f64());
 
         Ok(true)
@@ -778,7 +761,7 @@ where
         if !self.is_inserted {
             self.store
                 .metrics
-                .latency_insert_dropped
+                .op_duration_insert_dropped
                 .observe(self.duration.as_secs_f64());
             let mut filtered = false;
             if self.is_judged {
@@ -791,12 +774,12 @@ where
             if filtered {
                 self.store
                     .metrics
-                    .latency_insert_filtered
+                    .op_duration_insert_filtered
                     .observe(self.duration.as_secs_f64());
             } else {
                 self.store
                     .metrics
-                    .latency_insert_dropped
+                    .op_duration_insert_dropped
                     .observe(self.duration.as_secs_f64());
             }
         }
@@ -1196,6 +1179,7 @@ pub mod tests {
             vec![recorder.clone()];
 
         let config = TestStoreConfig {
+            name: "".to_string(),
             eviction_config: FifoConfig,
             device_config: FsDeviceConfig {
                 dir: PathBuf::from(tempdir.path()),
@@ -1213,7 +1197,6 @@ pub mod tests {
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
             event_listeners: vec![],
-            prometheus_config: PrometheusConfig::default(),
             clean_region_threshold: 1,
         };
 
@@ -1246,6 +1229,7 @@ pub mod tests {
         drop(store);
 
         let config = TestStoreConfig {
+            name: "".to_string(),
             eviction_config: FifoConfig,
             device_config: FsDeviceConfig {
                 dir: PathBuf::from(tempdir.path()),
@@ -1263,7 +1247,6 @@ pub mod tests {
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
             event_listeners: vec![],
-            prometheus_config: PrometheusConfig::default(),
             clean_region_threshold: 1,
         };
         let store = TestStore::open(config).await.unwrap();

From e3579ecda51214c4ba650da402bd7b1336df8916 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 4 Sep 2023 15:15:49 +0800
Subject: [PATCH 091/261] chore: export inner op metrics (#116)

* chore: export inner op metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* separate op and slow op duration

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/flusher.rs        |  2 +-
 foyer-storage/src/metrics.rs        | 49 ++++++++++++++++++++++++-----
 foyer-storage/src/reclaimer.rs      |  2 +-
 foyer-storage/src/region_manager.rs | 10 ++++++
 foyer-storage/src/store.rs          |  5 +--
 5 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index a79c3a39..02d2f0ba 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -80,7 +80,7 @@ where
     }
 
     async fn handle(&self, region_id: RegionId) -> Result<()> {
-        let _timer = self.metrics.op_duration_flush.start_timer();
+        let _timer = self.metrics.slow_op_duration_flush.start_timer();
 
         tracing::info!("[flusher] receive flush task, region: {}", region_id);
 
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index a4a853d5..f0e0f561 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -25,8 +25,11 @@ pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::defau
 #[derive(Debug)]
 pub struct GlobalMetrics {
     op_duration: HistogramVec,
+    slow_op_duration: HistogramVec,
     op_bytes: IntCounterVec,
     total_bytes: IntGaugeVec,
+
+    inner_op_duration: HistogramVec,
 }
 
 impl Default for GlobalMetrics {
@@ -45,6 +48,14 @@ impl GlobalMetrics {
         )
         .unwrap();
 
+        let slow_op_duration = register_histogram_vec!(
+            "foyer_storage_slow_op_duration",
+            "foyer storage slow op duration",
+            &["foyer", "op", "extra"],
+            vec![0.01, 0.1, 0.5, 0.77, 1.0, 2.5, 5.0, 7.5, 10.0],
+        )
+        .unwrap();
+
         let op_bytes = register_int_counter_vec!(
             "foyer_storage_op_bytes",
             "foyer storage op bytes",
@@ -55,10 +66,21 @@ impl GlobalMetrics {
         let total_bytes =
             register_int_gauge_vec!("total_bytes", "total bytes", &["foyer"]).unwrap();
 
+        let inner_op_duration = register_histogram_vec!(
+            "foyer_storage_inner_op_duration",
+            "foyer storage inner op duration",
+            &["foyer", "op", "extra"],
+            vec![0.000001, 0.00001, 0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
+        )
+        .unwrap();
+
         Self {
             op_duration,
+            slow_op_duration,
             op_bytes,
             total_bytes,
+
+            inner_op_duration,
         }
     }
 
@@ -75,8 +97,8 @@ pub struct Metrics {
     pub op_duration_lookup_hit: Histogram,
     pub op_duration_lookup_miss: Histogram,
     pub op_duration_remove: Histogram,
-    pub op_duration_flush: Histogram,
-    pub op_duration_reclaim: Histogram,
+    pub slow_op_duration_flush: Histogram,
+    pub slow_op_duration_reclaim: Histogram,
 
     pub op_bytes_insert: IntCounter,
     pub op_bytes_lookup: IntCounter,
@@ -85,6 +107,8 @@ pub struct Metrics {
     pub op_bytes_reinsert: IntCounter,
 
     pub total_bytes: IntGauge,
+
+    pub inner_op_duration_acquire_clean_region: Histogram,
 }
 
 impl Metrics {
@@ -105,9 +129,11 @@ impl Metrics {
             .op_duration
             .with_label_values(&[foyer, "lookup", "miss"]);
         let op_duration_remove = global.op_duration.with_label_values(&[foyer, "remove", ""]);
-        let op_duration_flush = global.op_duration.with_label_values(&[foyer, "flush", ""]);
-        let op_duration_reclaim = global
-            .op_duration
+        let slow_op_duration_flush = global
+            .slow_op_duration
+            .with_label_values(&[foyer, "flush", ""]);
+        let slow_op_duration_reclaim = global
+            .slow_op_duration
             .with_label_values(&[foyer, "reclaim", ""]);
 
         let op_bytes_insert = global.op_bytes.with_label_values(&[foyer, "insert", ""]);
@@ -118,6 +144,11 @@ impl Metrics {
 
         let total_bytes = global.total_bytes.with_label_values(&[foyer]);
 
+        let inner_op_duration_acquire_clean_region =
+            global
+                .inner_op_duration
+                .with_label_values(&[foyer, "acquire_clean_region", ""]);
+
         Self {
             op_duration_insert_inserted,
             op_duration_insert_filtered,
@@ -125,14 +156,18 @@ impl Metrics {
             op_duration_lookup_hit,
             op_duration_lookup_miss,
             op_duration_remove,
-            op_duration_flush,
-            op_duration_reclaim,
+            slow_op_duration_flush,
+            slow_op_duration_reclaim,
+
             op_bytes_insert,
             op_bytes_lookup,
             op_bytes_flush,
             op_bytes_reclaim,
             op_bytes_reinsert,
+
             total_bytes,
+
+            inner_op_duration_acquire_clean_region,
         }
     }
 }
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index da399ce4..0b7aabe5 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -112,7 +112,7 @@ where
             }
         };
 
-        let _timer = self.metrics.op_duration_reclaim.start_timer();
+        let _timer = self.metrics.slow_op_duration_reclaim.start_timer();
 
         let region = self.region_manager.region(&region_id);
 
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 37c9781c..0a1c5eaa 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -28,6 +28,7 @@ use tokio::sync::RwLock as AsyncRwLock;
 
 use crate::{
     device::Device,
+    metrics::Metrics,
     region::{AllocateResult, Region, RegionId},
 };
 
@@ -72,6 +73,8 @@ where
 
     /// Eviction policy.
     eviction: RwLock<EP>,
+
+    metrics: Arc<Metrics>,
 }
 
 impl<D, EP, EL> RegionManager<D, EP, EL>
@@ -85,6 +88,7 @@ where
         region_count: usize,
         eviction_config: EP::Config,
         device: D,
+        metrics: Arc<Metrics>,
     ) -> Self {
         let buffers = AsyncQueue::new();
         for _ in 0..buffer_count {
@@ -120,6 +124,7 @@ where
             regions,
             items,
             eviction: RwLock::new(eviction),
+            metrics,
         }
     }
 
@@ -160,7 +165,12 @@ where
         match self.rotate_batch.push(()) {
             Identity::Leader(rx) => {
                 // Wait a clean region to be released.
+                let timer = self
+                    .metrics
+                    .inner_op_duration_acquire_clean_region
+                    .start_timer();
                 let region_id = self.clean_regions.acquire().await;
+                drop(timer);
 
                 tracing::info!("switch to clean region: {}", region_id);
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 547af2c1..f71b847f 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -164,6 +164,8 @@ where
     pub async fn open(config: StoreConfig<K, V, D, EP>) -> Result<Arc<Self>> {
         tracing::info!("open store with config:\n{:#?}", config);
 
+        let metrics = Arc::new(METRICS.foyer(&config.name));
+
         let device = D::open(config.device_config).await?;
 
         let buffer_count = config.buffer_pool_size / device.region_size();
@@ -173,6 +175,7 @@ where
             device.regions(),
             config.eviction_config,
             device.clone(),
+            metrics.clone(),
         ));
 
         let indices = Arc::new(Indices::new(device.regions()));
@@ -187,8 +190,6 @@ where
             .map(|_| reclaimers_stop_tx.subscribe())
             .collect_vec();
 
-        let metrics = Arc::new(METRICS.foyer(&config.name));
-
         let store = Arc::new(Self {
             indices: indices.clone(),
             region_manager: region_manager.clone(),

From ecf8961ea6b2cfb210b410e76f80f575e4c67d81 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 4 Sep 2023 15:51:29 +0800
Subject: [PATCH 092/261] feat: integrate tracing framework (#118)

* feat: add tracing framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* ignore span log

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* pass shellcheck

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* remove too frequent span

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                            |  9 +++-
 foyer-storage-bench/Cargo.toml      | 10 +++++
 foyer-storage-bench/src/main.rs     | 64 +++++++++++++++++++++--------
 foyer-storage/src/region.rs         |  1 +
 foyer-storage/src/region_manager.rs | 50 ++++++++++++++--------
 foyer-storage/src/store.rs          |  4 +-
 foyer-workspace-hack/Cargo.toml     |  5 +++
 scripts/jaeger.sh                   | 20 +++++++++
 8 files changed, 126 insertions(+), 37 deletions(-)
 create mode 100755 scripts/jaeger.sh

diff --git a/Makefile b/Makefile
index 8928a890..34d8131b 100644
--- a/Makefile
+++ b/Makefile
@@ -5,11 +5,18 @@ deps:
 	./scripts/install-deps.sh
 
 check:
+	shellcheck ./scripts/*
 	cargo hakari generate
 	cargo hakari manage-deps
 	cargo sort -w
 	cargo fmt --all
+	cargo clippy --all-targets --features deadlock
+	cargo clippy --all-targets --features tokio-console
+	cargo clippy --all-targets --features trace
 	cargo clippy --all-targets
 
 test:
-	RUST_BACKTRACE=1 cargo test --all
\ No newline at end of file
+	RUST_BACKTRACE=1 cargo test --all
+
+jaeger:
+	./scripts/jaeger.sh
\ No newline at end of file
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index b8e83f4b..4ec0d77f 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -19,6 +19,9 @@ hdrhistogram = "7"
 itertools = "0.11.0"
 libc = "0.2"
 nix = { version = "0.27", features = ["fs", "mman"] }
+opentelemetry = { version = "0.20", features = ["rt-tokio"], optional = true }
+opentelemetry-otlp = { version = "0.13.0", optional = true }
+opentelemetry-semantic-conventions = { version = "0.12", optional = true }
 parking_lot = "0.12"
 rand = "0.8.5"
 rand_mt = "4.2.1"
@@ -32,8 +35,15 @@ tokio = { version = "1", features = [
     "signal",
 ] }
 tracing = "0.1"
+tracing-opentelemetry = { version = "0.21", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [features]
 deadlock = ["parking_lot/deadlock_detection", "foyer-storage/deadlock"]
 tokio-console = ["console-subscriber"]
+trace = [
+    "opentelemetry",
+    "opentelemetry-otlp",
+    "tracing-opentelemetry",
+    "opentelemetry-semantic-conventions",
+]
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 4880f65a..150284a5 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -150,26 +150,58 @@ type TStore = LfuFsStore<u64, Vec<u8>>;
 
 fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
+#[cfg(feature = "tokio-console")]
+fn init_logger() {
+    console_subscriber::init();
+}
+
+#[cfg(feature = "trace")]
+fn init_logger() {
+    use tracing::Level;
+    use tracing_subscriber::{filter::Targets, prelude::*};
+    let tracer = opentelemetry_otlp::new_pipeline()
+        .tracing()
+        .with_exporter(opentelemetry_otlp::new_exporter().tonic())
+        .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
+            opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
+                opentelemetry_semantic_conventions::resource::SERVICE_NAME,
+                "foyer-storage-bench",
+            )]),
+        ))
+        .install_batch(opentelemetry::runtime::Tokio)
+        .unwrap();
+    let opentelemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer);
+    tracing_subscriber::registry()
+        .with(
+            Targets::new()
+                .with_target("foyer_storage", Level::DEBUG)
+                .with_target("foyer_common", Level::DEBUG)
+                .with_target("foyer_intrusive", Level::DEBUG)
+                .with_target("foyer_storage_bench", Level::DEBUG),
+        )
+        .with(opentelemetry_layer)
+        .init();
+}
+
+#[cfg(not(any(feature = "tokio-console", feature = "trace")))]
+fn init_logger() {
+    use tracing_subscriber::{prelude::*, EnvFilter};
+
+    tracing_subscriber::registry()
+        .with(
+            tracing_subscriber::fmt::layer()
+                // .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
+                .with_line_number(true),
+        )
+        .with(EnvFilter::from_default_env())
+        .init();
+}
+
 #[tokio::main]
 async fn main() {
     is_send_sync_static::<TStore>();
 
-    #[cfg(feature = "tokio-console")]
-    console_subscriber::init();
-
-    #[cfg(not(feature = "tokio-console"))]
-    {
-        use tracing_subscriber::{prelude::*, EnvFilter};
-
-        tracing_subscriber::registry()
-            .with(
-                tracing_subscriber::fmt::layer()
-                    // .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
-                    .with_line_number(true),
-            )
-            .with(EnvFilter::from_default_env())
-            .init();
-    }
+    init_logger();
 
     #[cfg(feature = "deadlock")]
     {
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 1e8240c8..02215070 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -111,6 +111,7 @@ where
         }
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn allocate(&self, size: usize) -> AllocateResult {
         let future = {
             let inner = self.inner.clone();
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 0a1c5eaa..6a3efb0e 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -24,7 +24,8 @@ use foyer_intrusive::{
     intrusive_adapter, key_adapter,
 };
 use parking_lot::RwLock;
-use tokio::sync::RwLock as AsyncRwLock;
+use tokio::sync::Mutex as AsyncMutex;
+use tracing::Instrument;
 
 use crate::{
     device::Device,
@@ -56,7 +57,7 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    current: AsyncRwLock<Option<RegionId>>,
+    current: AsyncMutex<Option<RegionId>>,
     rotate_batch: Batch<(), ()>,
 
     /// Buffer pool for dirty buffers.
@@ -116,7 +117,7 @@ where
         }
 
         Self {
-            current: AsyncRwLock::new(None),
+            current: AsyncMutex::new(None),
             rotate_batch: Batch::new(),
             buffers,
             clean_regions,
@@ -142,25 +143,36 @@ where
         }
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn allocate_inner(&self, size: usize) -> AllocateResult {
-        let mut current = self.current.write().await;
-        if let Some(region_id) = *current {
-            let region = self.region(&region_id);
-            match region.allocate(size).await {
-                AllocateResult::Ok(slice) => AllocateResult::Ok(slice),
-                AllocateResult::Full { slice, remain } => {
-                    // current region is full, append dirty regions
-                    self.dirty_regions.release(region_id);
-                    *current = None;
-                    AllocateResult::Full { slice, remain }
+        let mut current = self
+            .current
+            .lock()
+            .instrument(tracing::debug_span!("lock_current"))
+            .await;
+
+        async {
+            if let Some(region_id) = *current {
+                let region = self.region(&region_id);
+                match region.allocate(size).await {
+                    AllocateResult::Ok(slice) => AllocateResult::Ok(slice),
+                    AllocateResult::Full { slice, remain } => {
+                        // current region is full, append dirty regions
+                        self.dirty_regions.release(region_id);
+                        *current = None;
+                        AllocateResult::Full { slice, remain }
+                    }
+                    AllocateResult::None => unreachable!(),
                 }
-                AllocateResult::None => unreachable!(),
+            } else {
+                AllocateResult::None
             }
-        } else {
-            AllocateResult::None
         }
+        .instrument(tracing::debug_span!("lock_current_zone"))
+        .await
     }
 
+    #[tracing::instrument(skip(self), fields(followers))]
     pub async fn rotate(&self) {
         match self.rotate_batch.push(()) {
             Identity::Leader(rx) => {
@@ -180,10 +192,13 @@ where
                 let buffer = self.buffers.acquire().await;
                 region.attach_buffer(buffer).await;
 
-                *self.current.write().await = Some(region_id);
+                {
+                    *self.current.lock().await = Some(region_id);
+                }
 
                 // Notify the batch to advance.
                 let items = self.rotate_batch.rotate();
+                tracing::Span::current().record("followers", items.len());
                 for item in items {
                     item.tx.send(()).unwrap();
                 }
@@ -193,7 +208,6 @@ where
         }
     }
 
-    #[tracing::instrument(skip(self))]
     pub fn region(&self, id: &RegionId) -> &Region<D> {
         &self.regions[*id as usize]
     }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index f71b847f..90639ee1 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -713,11 +713,11 @@ where
 
     /// Judge if the entry can be admitted by configured admission policies.
     pub fn judge(&mut self) -> bool {
-        let now = Instant::now();
         if !self.is_judged {
+            let now = Instant::now();
             self.store.judge_inner(self);
+            self.duration = now.elapsed();
         }
-        self.duration += now.elapsed();
         self.judges.judge()
     }
 
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index f7e7134c..c3e10154 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -18,12 +18,17 @@ crossbeam-utils = { version = "0.8" }
 either = { version = "1", default-features = false, features = ["use_std"] }
 futures-channel = { version = "0.3", features = ["sink"] }
 futures-core = { version = "0.3" }
+futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
+futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
 itertools = { version = "0.10" }
 nix = { version = "0.27", features = ["fs", "mman", "uio"] }
 parking_lot = { version = "0.12", features = ["deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
+regex = { version = "1" }
+regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
+regex-syntax = { version = "0.7" }
 tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 tracing-core = { version = "0.1" }
 
diff --git a/scripts/jaeger.sh b/scripts/jaeger.sh
new file mode 100755
index 00000000..433f3506
--- /dev/null
+++ b/scripts/jaeger.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+container="$(docker ps -q -f name=jaeger)"
+
+if [ "$container" ] ; then
+    echo "Stop jaeger..."
+    docker stop "$container" > /dev/null
+    docker rm "$container" > /dev/null
+else
+    echo "Start jaeger..."
+    docker run -d --name jaeger \
+        -p6831:6831/udp \
+        -p6832:6832/udp \
+        -p16686:16686 \
+        -p4317:4317 \
+        jaegertracing/all-in-one:latest \
+        --collector.otlp.enabled=true \
+        > /dev/null
+    echo "Browser jaeger via http://localhost:16686"
+fi
\ No newline at end of file

From eb0eb45151da9b496f751b668ec5913a56345c0f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 4 Sep 2023 16:24:55 +0800
Subject: [PATCH 093/261] feat: use sync lock in ErwLock instead of async lock
 (#119)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/Cargo.toml            |  2 +-
 foyer-storage/src/device/fs.rs      |  2 --
 foyer-storage/src/region.rs         | 44 ++++++++++++-----------
 foyer-storage/src/region_manager.rs | 54 +++++++++++------------------
 foyer-workspace-hack/Cargo.toml     |  3 +-
 5 files changed, 46 insertions(+), 59 deletions(-)

diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index f5556f92..690b5f7c 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -23,7 +23,7 @@ itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
 nix = { version = "0.27", features = ["fs", "mman", "uio"] }
-parking_lot = "0.12"
+parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 pin-project = "1"
 prometheus = "0.13"
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 6f8743c4..04061b1d 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -83,7 +83,6 @@ impl Device for FsDevice {
         Self::open(config).await
     }
 
-    #[tracing::instrument(level = "trace", skip(self, buf))]
     async fn write(
         &self,
         buf: impl IoBuf,
@@ -105,7 +104,6 @@ impl Device for FsDevice {
         Ok(res)
     }
 
-    #[tracing::instrument(level = "trace", skip(self, buf))]
     async fn read(
         &self,
         mut buf: impl IoBufMut,
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 02215070..a0c3f4ba 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -15,7 +15,9 @@
 use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, pin::Pin, sync::Arc, task::Waker};
 
 use futures::future::BoxFuture;
-use tokio::sync::{OwnedRwLockWriteGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
+use parking_lot::{
+    lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard,
+};
 use tracing::instrument;
 
 use crate::{
@@ -112,17 +114,17 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    pub async fn allocate(&self, size: usize) -> AllocateResult {
+    pub fn allocate(&self, size: usize) -> AllocateResult {
         let future = {
             let inner = self.inner.clone();
             async move {
-                let mut guard = inner.write().await;
+                let mut guard = inner.write();
                 guard.writers -= 1;
                 guard.wake_all();
             }
         };
 
-        let mut inner = self.inner.write().await;
+        let mut inner = self.inner.write();
 
         inner.writers += 1;
         let version = inner.version;
@@ -183,7 +185,7 @@ where
 
         // restrict guard lifetime
         {
-            let mut inner = self.inner.write().await;
+            let mut inner = self.inner.write();
 
             if version != 0 && version != inner.version {
                 return Ok(None);
@@ -198,7 +200,7 @@ where
                 let future = {
                     let inner = self.inner.clone();
                     async move {
-                        let mut guard = inner.write().await;
+                        let mut guard = inner.write();
                         guard.buffered_readers -= 1;
                         guard.wake_all();
                     }
@@ -234,7 +236,7 @@ where
                 .await?
                 != len
             {
-                let mut inner = self.inner.write().await;
+                let mut inner = self.inner.write();
                 inner.physical_readers -= 1;
                 inner.wake_all();
                 return Ok(None);
@@ -245,7 +247,7 @@ where
         let future = {
             let inner = self.inner.clone();
             async move {
-                let mut guard = inner.write().await;
+                let mut guard = inner.write();
                 guard.physical_readers -= 1;
                 guard.wake_all();
             }
@@ -257,7 +259,7 @@ where
     }
 
     pub async fn attach_buffer(&self, buf: Vec<u8, D::IoBufferAllocator>) {
-        let mut inner = self.inner.write().await;
+        let mut inner = self.inner.write();
 
         assert_eq!(inner.writers, 0);
         assert_eq!(inner.buffered_readers, 0);
@@ -266,13 +268,13 @@ where
     }
 
     pub async fn detach_buffer(&self) -> Vec<u8, D::IoBufferAllocator> {
-        let mut inner = self.inner.write().await;
+        let mut inner = self.inner.write();
 
         inner.detach_buffer()
     }
 
     pub async fn has_buffer(&self) -> bool {
-        let inner = self.inner.read().await;
+        let inner = self.inner.read();
         inner.has_buffer()
     }
 
@@ -282,7 +284,7 @@ where
         can_write: bool,
         can_buffered_read: bool,
         can_physical_read: bool,
-    ) -> OwnedRwLockWriteGuard<RegionInner<D::IoBufferAllocator>> {
+    ) -> ArcRwLockWriteGuard<RawRwLock, RegionInner<D::IoBufferAllocator>> {
         self.inner
             .exclusive(can_write, can_buffered_read, can_physical_read)
             .await
@@ -297,11 +299,11 @@ where
     }
 
     pub async fn version(&self) -> Version {
-        self.inner.read().await.version
+        self.inner.read().version
     }
 
     pub async fn advance(&self) -> Version {
-        let mut inner = self.inner.write().await;
+        let mut inner = self.inner.write();
         let res = inner.version;
         inner.version += 1;
         res
@@ -558,12 +560,12 @@ impl<A: BufferAllocator> ErwLock<A> {
         }
     }
 
-    pub async fn read(&self) -> RwLockReadGuard<'_, RegionInner<A>> {
-        self.inner.read().await
+    pub fn read(&self) -> RwLockReadGuard<'_, RegionInner<A>> {
+        self.inner.read()
     }
 
-    pub async fn write(&self) -> RwLockWriteGuard<'_, RegionInner<A>> {
-        self.inner.write().await
+    pub fn write(&self) -> RwLockWriteGuard<'_, RegionInner<A>> {
+        self.inner.write()
     }
 
     pub async fn exclusive(
@@ -571,10 +573,10 @@ impl<A: BufferAllocator> ErwLock<A> {
         can_write: bool,
         can_buffered_read: bool,
         can_physical_read: bool,
-    ) -> OwnedRwLockWriteGuard<RegionInner<A>> {
+    ) -> ArcRwLockWriteGuard<RawRwLock, RegionInner<A>> {
         loop {
             {
-                let guard = self.inner.clone().write_owned().await;
+                let guard = self.inner.clone().write_arc();
                 let is_ready = (can_write || guard.writers == 0)
                     && (can_buffered_read || guard.buffered_readers == 0)
                     && (can_physical_read || guard.physical_readers == 0);
@@ -582,7 +584,7 @@ impl<A: BufferAllocator> ErwLock<A> {
                     return guard;
                 }
             }
-            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+            tokio::time::sleep(std::time::Duration::from_millis(1)).await;
         }
     }
 }
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 6a3efb0e..93c1d07f 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -23,9 +23,7 @@ use foyer_intrusive::{
     eviction::{EvictionPolicy, EvictionPolicyExt},
     intrusive_adapter, key_adapter,
 };
-use parking_lot::RwLock;
-use tokio::sync::Mutex as AsyncMutex;
-use tracing::Instrument;
+use parking_lot::{Mutex, RwLock};
 
 use crate::{
     device::Device,
@@ -57,7 +55,7 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    current: AsyncMutex<Option<RegionId>>,
+    current: Mutex<Option<RegionId>>,
     rotate_batch: Batch<(), ()>,
 
     /// Buffer pool for dirty buffers.
@@ -117,7 +115,7 @@ where
         }
 
         Self {
-            current: AsyncMutex::new(None),
+            current: Mutex::new(None),
             rotate_batch: Batch::new(),
             buffers,
             clean_regions,
@@ -133,7 +131,7 @@ where
     #[tracing::instrument(skip(self))]
     pub async fn allocate(&self, size: usize, must_allocate: bool) -> AllocateResult {
         loop {
-            let res = self.allocate_inner(size).await;
+            let res = self.allocate_inner(size);
 
             if !must_allocate || !matches!(res, AllocateResult::None) {
                 return res;
@@ -143,36 +141,26 @@ where
         }
     }
 
-    #[tracing::instrument(skip(self))]
-    pub async fn allocate_inner(&self, size: usize) -> AllocateResult {
-        let mut current = self
-            .current
-            .lock()
-            .instrument(tracing::debug_span!("lock_current"))
-            .await;
-
-        async {
-            if let Some(region_id) = *current {
-                let region = self.region(&region_id);
-                match region.allocate(size).await {
-                    AllocateResult::Ok(slice) => AllocateResult::Ok(slice),
-                    AllocateResult::Full { slice, remain } => {
-                        // current region is full, append dirty regions
-                        self.dirty_regions.release(region_id);
-                        *current = None;
-                        AllocateResult::Full { slice, remain }
-                    }
-                    AllocateResult::None => unreachable!(),
+    pub fn allocate_inner(&self, size: usize) -> AllocateResult {
+        let mut current = self.current.lock();
+
+        if let Some(region_id) = *current {
+            let region = self.region(&region_id);
+            match region.allocate(size) {
+                AllocateResult::Ok(slice) => AllocateResult::Ok(slice),
+                AllocateResult::Full { slice, remain } => {
+                    // current region is full, append dirty regions
+                    self.dirty_regions.release(region_id);
+                    *current = None;
+                    AllocateResult::Full { slice, remain }
                 }
-            } else {
-                AllocateResult::None
+                AllocateResult::None => unreachable!(),
             }
+        } else {
+            AllocateResult::None
         }
-        .instrument(tracing::debug_span!("lock_current_zone"))
-        .await
     }
 
-    #[tracing::instrument(skip(self), fields(followers))]
     pub async fn rotate(&self) {
         match self.rotate_batch.push(()) {
             Identity::Leader(rx) => {
@@ -192,9 +180,7 @@ where
                 let buffer = self.buffers.acquire().await;
                 region.attach_buffer(buffer).await;
 
-                {
-                    *self.current.lock().await = Some(region_id);
-                }
+                *self.current.lock() = Some(region_id);
 
                 // Notify the batch to advance.
                 let items = self.rotate_batch.rotate();
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index c3e10154..8435893e 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -22,8 +22,9 @@ futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
 itertools = { version = "0.10" }
+lock_api = { version = "0.4", features = ["arc_lock"] }
 nix = { version = "0.27", features = ["fs", "mman", "uio"] }
-parking_lot = { version = "0.12", features = ["deadlock_detection"] }
+parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
 regex = { version = "1" }

From 5f8d42143eaab834c585392776281874e5f622b7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 5 Sep 2023 13:55:51 +0800
Subject: [PATCH 094/261] chore: enhance tracing (#122)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/analyze.rs  | 16 ++++++++++++++++
 foyer-storage-bench/src/main.rs     | 24 ++++++++++++++++++------
 foyer-storage/src/metrics.rs        | 14 ++++++++++++--
 foyer-storage/src/region_manager.rs | 15 +++++++++++++--
 scripts/jaeger.sh                   |  2 +-
 5 files changed, 60 insertions(+), 11 deletions(-)

diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index 34ab4b1a..02a9c234 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -57,15 +57,18 @@ pub struct Analysis {
     insert_lat_p50: u64,
     insert_lat_p90: u64,
     insert_lat_p99: u64,
+    insert_lat_pmax: u64,
 
     get_iops: f64,
     get_miss: f64,
     get_miss_lat_p50: u64,
     get_miss_lat_p90: u64,
     get_miss_lat_p99: u64,
+    get_miss_lat_pmax: u64,
     get_hit_lat_p50: u64,
     get_hit_lat_p90: u64,
     get_hit_lat_p99: u64,
+    get_hit_lat_pmax: u64,
 }
 
 #[derive(Default, Clone, Copy, Debug)]
@@ -75,15 +78,18 @@ pub struct MetricsDump {
     pub insert_lat_p50: u64,
     pub insert_lat_p90: u64,
     pub insert_lat_p99: u64,
+    pub insert_lat_pmax: u64,
 
     pub get_ios: usize,
     pub get_miss_ios: usize,
     pub get_hit_lat_p50: u64,
     pub get_hit_lat_p90: u64,
     pub get_hit_lat_p99: u64,
+    pub get_hit_lat_pmax: u64,
     pub get_miss_lat_p50: u64,
     pub get_miss_lat_p90: u64,
     pub get_miss_lat_p99: u64,
+    pub get_miss_lat_pmax: u64,
 }
 
 #[derive(Clone, Debug)]
@@ -124,21 +130,25 @@ impl Metrics {
         let insert_lats = self.insert_lats.read();
         let get_hit_lats = self.get_hit_lats.read();
         let get_miss_lats = self.get_miss_lats.read();
+
         MetricsDump {
             insert_ios: self.insert_ios.load(Ordering::Relaxed),
             insert_bytes: self.insert_bytes.load(Ordering::Relaxed),
             insert_lat_p50: insert_lats.value_at_quantile(0.5),
             insert_lat_p90: insert_lats.value_at_quantile(0.9),
             insert_lat_p99: insert_lats.value_at_quantile(0.99),
+            insert_lat_pmax: insert_lats.max(),
 
             get_ios: self.get_ios.load(Ordering::Relaxed),
             get_miss_ios: self.get_miss_ios.load(Ordering::Relaxed),
             get_hit_lat_p50: get_hit_lats.value_at_quantile(0.5),
             get_hit_lat_p90: get_hit_lats.value_at_quantile(0.9),
             get_hit_lat_p99: get_hit_lats.value_at_quantile(0.99),
+            get_hit_lat_pmax: get_hit_lats.max(),
             get_miss_lat_p50: get_miss_lats.value_at_quantile(0.5),
             get_miss_lat_p90: get_miss_lats.value_at_quantile(0.9),
             get_miss_lat_p99: get_miss_lats.value_at_quantile(0.99),
+            get_miss_lat_pmax: get_miss_lats.max(),
         }
     }
 }
@@ -184,6 +194,7 @@ impl std::fmt::Display for Analysis {
         writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
         writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
         writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
+        writeln!(f, "insert lat pmax: {}us", self.insert_lat_pmax)?;
 
         // get statics
         writeln!(f, "get iops: {:.1}/s", self.get_iops)?;
@@ -191,9 +202,11 @@ impl std::fmt::Display for Analysis {
         writeln!(f, "get hit lat p50: {}us", self.get_hit_lat_p50)?;
         writeln!(f, "get hit lat p90: {}us", self.get_hit_lat_p90)?;
         writeln!(f, "get hit lat p99: {}us", self.get_hit_lat_p99)?;
+        writeln!(f, "get hit lat pmax: {}us", self.get_hit_lat_pmax)?;
         writeln!(f, "get miss lat p50: {}us", self.get_miss_lat_p50)?;
         writeln!(f, "get miss lat p90: {}us", self.get_miss_lat_p90)?;
         writeln!(f, "get miss lat p99: {}us", self.get_miss_lat_p99)?;
+        writeln!(f, "get miss lat pmax: {}us", self.get_miss_lat_pmax)?;
 
         Ok(())
     }
@@ -233,15 +246,18 @@ pub fn analyze(
         insert_lat_p50: metrics_dump_end.insert_lat_p50,
         insert_lat_p90: metrics_dump_end.insert_lat_p90,
         insert_lat_p99: metrics_dump_end.insert_lat_p99,
+        insert_lat_pmax: metrics_dump_end.insert_lat_pmax,
 
         get_iops,
         get_miss,
         get_hit_lat_p50: metrics_dump_end.get_hit_lat_p50,
         get_hit_lat_p90: metrics_dump_end.get_hit_lat_p90,
         get_hit_lat_p99: metrics_dump_end.get_hit_lat_p99,
+        get_hit_lat_pmax: metrics_dump_end.get_hit_lat_pmax,
         get_miss_lat_p50: metrics_dump_end.get_miss_lat_p50,
         get_miss_lat_p90: metrics_dump_end.get_miss_lat_p90,
         get_miss_lat_p99: metrics_dump_end.get_miss_lat_p99,
+        get_miss_lat_pmax: metrics_dump_end.get_miss_lat_pmax,
     }
 }
 
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 150284a5..f0f678a1 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -157,17 +157,29 @@ fn init_logger() {
 
 #[cfg(feature = "trace")]
 fn init_logger() {
+    use opentelemetry::sdk::{
+        trace::{BatchConfig, Config},
+        Resource,
+    };
+    use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
     use tracing::Level;
     use tracing_subscriber::{filter::Targets, prelude::*};
+
+    let trace_config =
+        Config::default().with_resource(Resource::new(vec![opentelemetry::KeyValue::new(
+            SERVICE_NAME,
+            "foyer-storage-bench",
+        )]));
+    let batch_config = BatchConfig::default()
+        .with_max_queue_size(1048576)
+        .with_max_export_batch_size(1024)
+        .with_max_concurrent_exports(4);
+
     let tracer = opentelemetry_otlp::new_pipeline()
         .tracing()
         .with_exporter(opentelemetry_otlp::new_exporter().tonic())
-        .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
-            opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
-                opentelemetry_semantic_conventions::resource::SERVICE_NAME,
-                "foyer-storage-bench",
-            )]),
-        ))
+        .with_trace_config(trace_config)
+        .with_batch_config(batch_config)
         .install_batch(opentelemetry::runtime::Tokio)
         .unwrap();
     let opentelemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer);
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index f0e0f561..452d2d71 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -63,8 +63,12 @@ impl GlobalMetrics {
         )
         .unwrap();
 
-        let total_bytes =
-            register_int_gauge_vec!("total_bytes", "total bytes", &["foyer"]).unwrap();
+        let total_bytes = register_int_gauge_vec!(
+            "foyer_storage_total_bytes",
+            "foyer storage total bytes",
+            &["foyer"]
+        )
+        .unwrap();
 
         let inner_op_duration = register_histogram_vec!(
             "foyer_storage_inner_op_duration",
@@ -109,6 +113,7 @@ pub struct Metrics {
     pub total_bytes: IntGauge,
 
     pub inner_op_duration_acquire_clean_region: Histogram,
+    pub inner_op_duration_acquire_clean_buffer: Histogram,
 }
 
 impl Metrics {
@@ -148,6 +153,10 @@ impl Metrics {
             global
                 .inner_op_duration
                 .with_label_values(&[foyer, "acquire_clean_region", ""]);
+        let inner_op_duration_acquire_clean_buffer =
+            global
+                .inner_op_duration
+                .with_label_values(&[foyer, "acquire_clean_buffer", ""]);
 
         Self {
             op_duration_insert_inserted,
@@ -168,6 +177,7 @@ impl Metrics {
             total_bytes,
 
             inner_op_duration_acquire_clean_region,
+            inner_op_duration_acquire_clean_buffer,
         }
     }
 }
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 93c1d07f..63808a82 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -24,6 +24,7 @@ use foyer_intrusive::{
     intrusive_adapter, key_adapter,
 };
 use parking_lot::{Mutex, RwLock};
+use tracing::Instrument;
 
 use crate::{
     device::Device,
@@ -141,6 +142,7 @@ where
         }
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn allocate_inner(&self, size: usize) -> AllocateResult {
         let mut current = self.current.lock();
 
@@ -161,6 +163,7 @@ where
         }
     }
 
+    #[tracing::instrument(skip(self))]
     pub async fn rotate(&self) {
         match self.rotate_batch.push(()) {
             Identity::Leader(rx) => {
@@ -169,7 +172,11 @@ where
                     .metrics
                     .inner_op_duration_acquire_clean_region
                     .start_timer();
-                let region_id = self.clean_regions.acquire().await;
+                let region_id = self
+                    .clean_regions
+                    .acquire()
+                    .instrument(tracing::debug_span!("acquire_clean_region"))
+                    .await;
                 drop(timer);
 
                 tracing::info!("switch to clean region: {}", region_id);
@@ -177,7 +184,11 @@ where
                 let region = self.region(&region_id);
                 region.advance().await;
 
-                let buffer = self.buffers.acquire().await;
+                let buffer = self
+                    .buffers
+                    .acquire()
+                    .instrument(tracing::debug_span!("acquire_clean_buffer"))
+                    .await;
                 region.attach_buffer(buffer).await;
 
                 *self.current.lock() = Some(region_id);
diff --git a/scripts/jaeger.sh b/scripts/jaeger.sh
index 433f3506..09c103d1 100755
--- a/scripts/jaeger.sh
+++ b/scripts/jaeger.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-container="$(docker ps -q -f name=jaeger)"
+container="$(docker ps -a -q -f name=jaeger)"
 
 if [ "$container" ] ; then
     echo "Stop jaeger..."

From 7885ee96dbce0bccaff1ffcd91b31e4e54a820f4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 5 Sep 2023 14:31:38 +0800
Subject: [PATCH 095/261] refactor: remove async slice drop (#123)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                     |   5 +-
 foyer-storage/src/flusher.rs |   2 +-
 foyer-storage/src/region.rs  | 118 ++++++++++-------------------------
 foyer-storage/src/store.rs   |  20 +++---
 scripts/install-deps.sh      |   2 +-
 5 files changed, 46 insertions(+), 101 deletions(-)

diff --git a/Makefile b/Makefile
index 34d8131b..0c9ba321 100644
--- a/Makefile
+++ b/Makefile
@@ -16,7 +16,8 @@ check:
 	cargo clippy --all-targets
 
 test:
-	RUST_BACKTRACE=1 cargo test --all
-
+	RUST_BACKTRACE=1 cargo nextest run --all
+	RUST_BACKTRACE=1 cargo test --doc
+	
 jaeger:
 	./scripts/jaeger.sh
\ No newline at end of file
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 02d2f0ba..2f98cc90 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -114,7 +114,7 @@ where
                 .await?;
             offset += len;
         }
-        slice.destroy().await;
+        drop(slice);
 
         tracing::trace!("[flusher] step 2");
 
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index a0c3f4ba..9f485202 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -12,9 +12,8 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, pin::Pin, sync::Arc, task::Waker};
+use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, sync::Arc, task::Waker};
 
-use futures::future::BoxFuture;
 use parking_lot::{
     lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard,
 };
@@ -115,13 +114,14 @@ where
 
     #[tracing::instrument(skip(self))]
     pub fn allocate(&self, size: usize) -> AllocateResult {
-        let future = {
+        let cleanup = {
             let inner = self.inner.clone();
-            async move {
+            let f = move || {
                 let mut guard = inner.write();
                 guard.writers -= 1;
                 guard.wake_all();
-            }
+            };
+            Box::new(f)
         };
 
         let mut inner = self.inner.write();
@@ -146,7 +146,7 @@ where
                 region_id,
                 version,
                 offset,
-                future: Some(Box::pin(future)),
+                cleanup: Some(cleanup),
             };
             AllocateResult::Full { slice, remain }
         } else {
@@ -160,7 +160,7 @@ where
                 region_id,
                 version,
                 offset,
-                future: Some(Box::pin(future)),
+                cleanup: Some(cleanup),
             };
             AllocateResult::Ok(slice)
         }
@@ -197,18 +197,19 @@ where
                 inner.buffered_readers += 1;
                 let allocator = inner.buffer.as_ref().unwrap().allocator().clone();
                 let slice = unsafe { Slice::new(&inner.buffer.as_ref().unwrap()[start..end]) };
-                let future = {
+                let cleanup = {
                     let inner = self.inner.clone();
-                    async move {
+                    let f = move || {
                         let mut guard = inner.write();
                         guard.buffered_readers -= 1;
                         guard.wake_all();
-                    }
+                    };
+                    Box::new(f)
                 };
                 return Ok(Some(ReadSlice::Slice {
                     slice,
                     allocator: Some(allocator),
-                    future: Some(Box::pin(future)),
+                    cleanup: Some(cleanup),
                 }));
             }
 
@@ -244,17 +245,18 @@ where
             offset += len;
         }
 
-        let future = {
+        let cleanup = {
             let inner = self.inner.clone();
-            async move {
+            let f = move || {
                 let mut guard = inner.write();
                 guard.physical_readers -= 1;
                 guard.wake_all();
-            }
+            };
+            Box::new(f)
         };
         Ok(Some(ReadSlice::Owned {
             buf: Some(buf),
-            future: Some(Box::pin(future)),
+            cleanup: Some(cleanup),
         }))
     }
 
@@ -351,14 +353,14 @@ where
 
 // read & write slice
 
-#[pin_project::pin_project(project = WriteSliceProj, PinnedDrop)]
+pub trait CleanupFn = FnOnce() + Send + Sync + 'static;
+
 pub struct WriteSlice {
     slice: SliceMut,
     region_id: RegionId,
     version: Version,
     offset: usize,
-    #[pin]
-    future: Option<BoxFuture<'static, ()>>,
+    cleanup: Option<Box<dyn CleanupFn>>,
 }
 
 impl Debug for WriteSlice {
@@ -392,15 +394,6 @@ impl WriteSlice {
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
-
-    /// # Safety
-    ///
-    /// `destroy` MUST be called before actually drop if `future` is set.
-    pub async fn destroy(mut self) {
-        if let Some(future) = self.future.take() {
-            future.await;
-        }
-    }
 }
 
 impl AsRef<[u8]> for WriteSlice {
@@ -415,29 +408,14 @@ impl AsMut<[u8]> for WriteSlice {
     }
 }
 
-#[pin_project::pinned_drop]
-impl PinnedDrop for WriteSlice {
-    fn drop(self: Pin<&mut Self>) {
-        let mut this = self.project();
-        if let Some(future) = this.future.take() {
-            tracing::error!("future is not consumed. This may be caused by error early return. If there's not, check if there's slice not destroyed. {:?}", this);
-            tokio::spawn(future);
+impl Drop for WriteSlice {
+    fn drop(&mut self) {
+        if let Some(f) = self.cleanup.take() {
+            f()
         }
     }
 }
 
-impl<'pin> Debug for WriteSliceProj<'pin> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("WriteSlice")
-            .field("slice", &self.slice)
-            .field("region_id", &self.region_id)
-            .field("version", &self.version)
-            .field("offset", &self.offset)
-            .finish()
-    }
-}
-
-#[pin_project::pin_project(project = ReadSliceProj, PinnedDrop)]
 pub enum ReadSlice<A>
 where
     A: BufferAllocator,
@@ -445,13 +423,11 @@ where
     Slice {
         slice: Slice,
         allocator: Option<A>,
-        #[pin]
-        future: Option<BoxFuture<'static, ()>>,
+        cleanup: Option<Box<dyn CleanupFn>>,
     },
     Owned {
         buf: Option<Vec<u8, A>>,
-        #[pin]
-        future: Option<BoxFuture<'static, ()>>,
+        cleanup: Option<Box<dyn CleanupFn>>,
     },
 }
 
@@ -490,18 +466,6 @@ where
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
-
-    /// # Safety
-    ///
-    /// `destroy` MUST be called before actually drop if `future` is set.
-    pub async fn destroy(mut self) {
-        if let Some(future) = match &mut self {
-            ReadSlice::Slice { future, .. } => future.take(),
-            ReadSlice::Owned { future, .. } => future.take(),
-        } {
-            future.await;
-        }
-    }
 }
 
 impl<A> AsRef<[u8]> for ReadSlice<A>
@@ -516,34 +480,16 @@ where
     }
 }
 
-#[pin_project::pinned_drop]
-impl<A> PinnedDrop for ReadSlice<A>
+impl<A> Drop for ReadSlice<A>
 where
     A: BufferAllocator,
 {
-    fn drop(self: Pin<&mut Self>) {
-        let mut this = self.project();
-        if let Some(future) = match &mut this {
-            ReadSliceProj::Slice { future, .. } => future.take(),
-            ReadSliceProj::Owned { future, .. } => future.take(),
+    fn drop(&mut self) {
+        if let Some(f) = match self {
+            ReadSlice::Slice { cleanup, .. } => cleanup.take(),
+            ReadSlice::Owned { cleanup, .. } => cleanup.take(),
         } {
-            tracing::error!("future is not consumed. This may be caused by error early return. If there's not, check if there's slice not destroyed. {:?}", this);
-            tokio::spawn(future);
-        }
-    }
-}
-
-impl<'pin, A: BufferAllocator> Debug for ReadSliceProj<'pin, A> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::Slice {
-                slice, allocator, ..
-            } => f
-                .debug_struct("Slice")
-                .field("slice", slice)
-                .field("allocator", allocator)
-                .finish(),
-            Self::Owned { buf, .. } => f.debug_struct("Owned").field("buf", buf).finish(),
+            f();
         }
     }
 }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 90639ee1..944b371a 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -436,7 +436,7 @@ where
                 Ok(None)
             }
         };
-        slice.destroy().await;
+        drop(slice);
 
         self.metrics
             .op_duration_lookup_hit
@@ -500,11 +500,10 @@ where
                     padding: remain as u64,
                 };
                 footer.write(slice.as_mut());
-                slice.destroy().await;
             }
             AllocateResult::Ok(slice) => {
                 // region is empty, skip
-                slice.destroy().await
+                drop(slice);
             }
             AllocateResult::None => unreachable!(),
         }
@@ -632,7 +631,7 @@ where
                         padding: remain as u64,
                     };
                     footer.write(slice.as_mut());
-                    slice.destroy().await;
+                    drop(slice);
                 }
                 AllocateResult::None => return Ok(false),
             }
@@ -650,8 +649,7 @@ where
 
             key: key.clone(),
         };
-
-        slice.destroy().await;
+        drop(slice);
 
         self.indices.insert(index);
 
@@ -958,7 +956,7 @@ where
         };
 
         let footer = RegionFooter::read(slice.as_ref());
-        slice.destroy().await;
+        drop(slice);
 
         if footer.magic != REGION_MAGIC {
             return Ok(None);
@@ -1000,16 +998,16 @@ where
                 align - EntryFooter::serialized_len() - (footer.padding + footer.key_len) as usize;
             let rel_end = align - EntryFooter::serialized_len() - footer.padding as usize;
             let key = K::read(&slice.as_ref()[rel_start..rel_end]);
-            slice.destroy().await;
+            drop(slice);
             key
         } else {
-            slice.destroy().await;
+            drop(slice);
             let s = self.region.load(align_start..align_end, 0).await?.unwrap();
             let rel_start = abs_start - align_start;
             let rel_end = abs_end - align_start;
 
             let key = K::read(&s.as_ref()[rel_start..rel_end]);
-            s.destroy().await;
+            drop(s);
             key
         };
 
@@ -1038,7 +1036,7 @@ where
         let end = start + index.len as usize;
         let slice = self.region.load(start..end, 0).await?.unwrap();
         let kv = read_entry::<K, V>(slice.as_ref());
-        slice.destroy().await;
+        drop(slice);
 
         Ok(kv)
     }
diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh
index 357896a6..bf0869ae 100755
--- a/scripts/install-deps.sh
+++ b/scripts/install-deps.sh
@@ -4,4 +4,4 @@ if [ -z "$(which cargo-binstall)" ]; then
     curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 fi
 
-cargo binstall -y cargo-hakari cargo-sort
\ No newline at end of file
+cargo binstall -y cargo-hakari cargo-sort cargo-nextest
\ No newline at end of file

From 2b8907c3cc629da22622a17bbc6bf16dc52c7bc1 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 6 Sep 2023 14:30:45 +0800
Subject: [PATCH 096/261] chore: export metrics registry (#124)

* feat: export metrics registry

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* export directly by static

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* support set registry

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/metrics.rs | 41 +++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 452d2d71..8a65e575 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -12,13 +12,25 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::LazyLock;
+use std::sync::{LazyLock, OnceLock};
 
 use prometheus::{
-    register_histogram_vec, register_int_counter_vec, register_int_gauge_vec, Histogram,
-    HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
+    register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
+    register_int_gauge_vec_with_registry, Histogram, HistogramVec, IntCounter, IntCounterVec,
+    IntGauge, IntGaugeVec, Registry,
 };
 
+pub static REGISTRY: OnceLock<Registry> = OnceLock::new();
+
+/// Set metrics registry for `foyer`.
+///
+/// Metrics registry must be set before `open`.
+///
+/// Return `true` if set succeeds.
+pub fn set_metrics_registry(registry: Registry) -> bool {
+    REGISTRY.set(registry).is_ok()
+}
+
 /// Multiple foyer instance will share the same global metrics with different label `foyer` name.
 pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
 
@@ -34,47 +46,52 @@ pub struct GlobalMetrics {
 
 impl Default for GlobalMetrics {
     fn default() -> Self {
-        Self::new()
+        Self::new(REGISTRY.get_or_init(|| prometheus::default_registry().clone()))
     }
 }
 
 impl GlobalMetrics {
-    pub fn new() -> Self {
-        let op_duration = register_histogram_vec!(
+    pub fn new(registry: &Registry) -> Self {
+        let op_duration = register_histogram_vec_with_registry!(
             "foyer_storage_op_duration",
             "foyer storage op duration",
             &["foyer", "op", "extra"],
             vec![0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
+            registry,
         )
         .unwrap();
 
-        let slow_op_duration = register_histogram_vec!(
+        let slow_op_duration = register_histogram_vec_with_registry!(
             "foyer_storage_slow_op_duration",
             "foyer storage slow op duration",
             &["foyer", "op", "extra"],
             vec![0.01, 0.1, 0.5, 0.77, 1.0, 2.5, 5.0, 7.5, 10.0],
+            registry,
         )
         .unwrap();
 
-        let op_bytes = register_int_counter_vec!(
+        let op_bytes = register_int_counter_vec_with_registry!(
             "foyer_storage_op_bytes",
             "foyer storage op bytes",
-            &["foyer", "op", "extra"]
+            &["foyer", "op", "extra"],
+            registry,
         )
         .unwrap();
 
-        let total_bytes = register_int_gauge_vec!(
+        let total_bytes = register_int_gauge_vec_with_registry!(
             "foyer_storage_total_bytes",
             "foyer storage total bytes",
-            &["foyer"]
+            &["foyer"],
+            registry,
         )
         .unwrap();
 
-        let inner_op_duration = register_histogram_vec!(
+        let inner_op_duration = register_histogram_vec_with_registry!(
             "foyer_storage_inner_op_duration",
             "foyer storage inner op duration",
             &["foyer", "op", "extra"],
             vec![0.000001, 0.00001, 0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
+            registry,
         )
         .unwrap();
 

From eed92cc833c0afc628a04431006ee8e3187ae8b4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 6 Sep 2023 21:20:22 +0800
Subject: [PATCH 097/261] refactor: refactor cache file format, use header
 instead of footer (#125)

---
 foyer-common/src/code.rs             |   4 +-
 foyer-common/src/lib.rs              |   1 +
 foyer-storage-bench/src/main.rs      |   4 +-
 foyer-storage/src/admission/mod.rs   |   2 +-
 foyer-storage/src/device/fs.rs       |   1 -
 foyer-storage/src/event.rs           |   2 +-
 foyer-storage/src/reclaimer.rs       |   8 +-
 foyer-storage/src/region.rs          |  27 ++++
 foyer-storage/src/region_manager.rs  |  22 ++-
 foyer-storage/src/reinsertion/mod.rs |   2 +-
 foyer-storage/src/store.rs           | 234 +++++++++++----------------
 11 files changed, 139 insertions(+), 168 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 3b7e4811..d73dfa41 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -15,7 +15,7 @@
 use bytes::{Buf, BufMut};
 use paste::paste;
 
-#[allow(unused_variables)]
+#[expect(unused_variables)]
 pub trait Key:
     Sized
     + Send
@@ -50,7 +50,7 @@ pub trait Key:
     }
 }
 
-#[allow(unused_variables)]
+#[expect(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
     #[cfg_attr(coverage_nightly, no_coverage)]
     fn weight(&self) -> usize {
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index e12637d6..8b8e5e1a 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -13,6 +13,7 @@
 //  limitations under the License.
 
 #![feature(trait_alias)]
+#![feature(lint_reasons)]
 #![cfg_attr(coverage_nightly, feature(no_coverage))]
 
 pub mod batch;
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index f0f678a1..83ea28de 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -13,6 +13,7 @@
 //  limitations under the License.
 
 #![feature(let_chains)]
+#![feature(lint_reasons)]
 
 mod analyze;
 mod rate;
@@ -410,7 +411,6 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop_tx: broadc
     join_all(r_handles).await;
 }
 
-#[allow(clippy::too_many_arguments)]
 async fn write(
     entry_size: usize,
     rate: Option<f64>,
@@ -454,7 +454,7 @@ async fn write(
     }
 }
 
-#[allow(clippy::too_many_arguments)]
+#[expect(clippy::too_many_arguments)]
 async fn read(
     entry_size: usize,
     rate: Option<f64>,
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 54fb2092..5a806abe 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -18,7 +18,7 @@ use std::{fmt::Debug, sync::Arc};
 
 use crate::{indices::Indices, metrics::Metrics};
 
-#[allow(unused_variables)]
+#[expect(unused_variables)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 04061b1d..3c7ead57 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -61,7 +61,6 @@ impl FsDeviceConfig {
 struct FsDeviceInner {
     config: FsDeviceConfig,
 
-    #[allow(dead_code)]
     dir: File,
 
     files: Vec<File>,
diff --git a/foyer-storage/src/event.rs b/foyer-storage/src/event.rs
index 0b00f4a7..60cbd624 100644
--- a/foyer-storage/src/event.rs
+++ b/foyer-storage/src/event.rs
@@ -19,7 +19,7 @@ use foyer_common::code::{Key, Value};
 
 use crate::error::Result;
 
-#[allow(unused_variables)]
+#[expect(unused_variables)]
 #[async_trait]
 pub trait EventListener: Send + Sync + 'static + Debug {
     type K: Key;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 0b7aabe5..ad882a92 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -197,19 +197,15 @@ where
             }
         };
 
-        if  !self.store.reinsertions().is_empty() && let Err(e) = reinsert().await {
+        if !self.store.reinsertions().is_empty() && let Err(e) = reinsert().await {
             tracing::warn!("reinsert region {:?} error: {:?}", region, e);
         }
 
         // step 3: set region last block zero
         let align = region.device().align();
-        let region_size = region.device().region_size();
         let mut buf = region.device().io_buffer(align, align);
         (&mut buf[..]).put_slice(&vec![0; align]);
-        region
-            .device()
-            .write(buf, region_id, (region_size - align) as u64, align)
-            .await?;
+        region.device().write(buf, region_id, 0, align).await?;
 
         // step 4: send clean region
         self.region_manager.clean_regions().release(region_id);
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 9f485202..0a6fc92f 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -14,6 +14,7 @@
 
 use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, sync::Arc, task::Waker};
 
+use bytes::{Buf, BufMut};
 use parking_lot::{
     lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard,
 };
@@ -29,6 +30,7 @@ pub type RegionId = u32;
 /// 0 matches any version
 pub type Version = u32;
 
+#[derive(Debug)]
 pub enum AllocateResult {
     Ok(WriteSlice),
     Full { slice: WriteSlice, remain: usize },
@@ -45,6 +47,25 @@ impl AllocateResult {
     }
 }
 
+pub const REGION_MAGIC: u64 = 0x19970327;
+
+#[derive(Debug)]
+pub struct RegionHeader {
+    /// magic number to decide a valid region
+    pub magic: u64,
+}
+
+impl RegionHeader {
+    pub fn write(&self, buf: &mut [u8]) {
+        (&mut buf[..]).put_u64(self.magic);
+    }
+
+    pub fn read(buf: &[u8]) -> Self {
+        let magic = (&buf[..]).get_u64();
+        Self { magic }
+    }
+}
+
 #[derive(Debug)]
 pub struct RegionInner<A>
 where
@@ -267,6 +288,12 @@ where
         assert_eq!(inner.buffered_readers, 0);
 
         inner.attach_buffer(buf);
+        let buffer = inner.buffer.as_deref_mut().unwrap();
+        let header = RegionHeader {
+            magic: REGION_MAGIC,
+        };
+        header.write(buffer);
+        inner.len = self.device.align();
     }
 
     pub async fn detach_buffer(&self) -> Vec<u8, D::IoBufferAllocator> {
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 63808a82..b9c31614 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -29,7 +29,7 @@ use tracing::Instrument;
 use crate::{
     device::Device,
     metrics::Metrics,
-    region::{AllocateResult, Region, RegionId},
+    region::{Region, RegionId, WriteSlice},
 };
 
 #[derive(Debug)]
@@ -130,11 +130,11 @@ where
 
     /// Allocate a buffer slice with given size in an active region to write.
     #[tracing::instrument(skip(self))]
-    pub async fn allocate(&self, size: usize, must_allocate: bool) -> AllocateResult {
+    pub async fn allocate(&self, size: usize, must_allocate: bool) -> Option<WriteSlice> {
         loop {
             let res = self.allocate_inner(size);
 
-            if !must_allocate || !matches!(res, AllocateResult::None) {
+            if res.is_some() || !must_allocate {
                 return res;
             }
 
@@ -143,23 +143,21 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn allocate_inner(&self, size: usize) -> AllocateResult {
+    pub fn allocate_inner(&self, size: usize) -> Option<WriteSlice> {
         let mut current = self.current.lock();
 
         if let Some(region_id) = *current {
-            let region = self.region(&region_id);
-            match region.allocate(size) {
-                AllocateResult::Ok(slice) => AllocateResult::Ok(slice),
-                AllocateResult::Full { slice, remain } => {
-                    // current region is full, append dirty regions
+            match self.region(&region_id).allocate(size) {
+                crate::region::AllocateResult::Ok(slice) => Some(slice),
+                crate::region::AllocateResult::Full { .. } => {
                     self.dirty_regions.release(region_id);
                     *current = None;
-                    AllocateResult::Full { slice, remain }
+                    None
                 }
-                AllocateResult::None => unreachable!(),
+                crate::region::AllocateResult::None => None,
             }
         } else {
-            AllocateResult::None
+            None
         }
     }
 
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index 006301ce..448bdc39 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -17,7 +17,7 @@ use foyer_common::code::{Key, Value};
 use crate::{indices::Indices, metrics::Metrics};
 use std::{fmt::Debug, sync::Arc};
 
-#[allow(unused_variables)]
+#[expect(unused_variables)]
 pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 944b371a..464dba34 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -38,7 +38,7 @@ use crate::{
     judge::Judges,
     metrics::{Metrics, METRICS},
     reclaimer::Reclaimer,
-    region::{AllocateResult, Region, RegionId},
+    region::{Region, RegionHeader, RegionId, REGION_MAGIC},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
 };
@@ -46,7 +46,6 @@ use foyer_common::code::{Key, Value};
 use foyer_intrusive::core::adapter::Link;
 use std::hash::Hasher;
 
-const REGION_MAGIC: u64 = 0x19970327;
 const DEFAULT_BROADCAST_CAPACITY: usize = 4096;
 
 pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
@@ -431,7 +430,7 @@ where
         let res = match read_entry::<K, V>(slice.as_ref()) {
             Some((_key, value)) => Ok(Some(value)),
             None => {
-                // Remove index if the storage layer fails to lookup it (because of region version mismatch).
+                // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
                 self.indices.remove(key);
                 Ok(None)
             }
@@ -483,30 +482,16 @@ where
 
     fn serialized_len(&self, key: &K, value: &V) -> usize {
         let unaligned =
-            key.serialized_len() + value.serialized_len() + EntryFooter::serialized_len();
+            EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len();
         bits::align_up(self.device.align(), unaligned)
     }
 
     async fn seal(&self) -> Result<()> {
-        match self
-            .region_manager
-            .allocate(self.device.region_size() - self.device.align(), true)
-            .await
-        {
-            AllocateResult::Full { mut slice, remain } => {
-                // current region is full, write region footer and try allocate again
-                let footer = RegionFooter {
-                    magic: REGION_MAGIC,
-                    padding: remain as u64,
-                };
-                footer.write(slice.as_mut());
-            }
-            AllocateResult::Ok(slice) => {
-                // region is empty, skip
-                drop(slice);
-            }
-            AllocateResult::None => unreachable!(),
-        }
+        // Try allocate the max size of a region to trigger flush.
+        // `max size == region size - region align` (first align block is reserved for region header)
+        self.region_manager
+            .allocate(self.device.region_size() - self.device.align(), false)
+            .await;
         Ok(())
     }
 
@@ -608,6 +593,7 @@ where
         }
 
         let serialized_len = self.serialized_len(key, &value);
+
         if key.serialized_len() + value.serialized_len() != writer.weight {
             tracing::error!(
                 "weight != key.serialized_len() + value.serialized_len(), weight: {}, key size: {}, value size: {}, key: {:?}",
@@ -617,24 +603,14 @@ where
 
         self.metrics.op_bytes_insert.inc_by(serialized_len as u64);
 
-        let mut slice = loop {
-            match self
-                .region_manager
-                .allocate(serialized_len, !writer.is_skippable)
-                .await
-            {
-                AllocateResult::Ok(slice) => break slice,
-                AllocateResult::Full { mut slice, remain } => {
-                    // current region is full, write region footer and try allocate again
-                    let footer = RegionFooter {
-                        magic: REGION_MAGIC,
-                        padding: remain as u64,
-                    };
-                    footer.write(slice.as_mut());
-                    drop(slice);
-                }
-                AllocateResult::None => return Ok(false),
-            }
+        let mut slice = match self
+            .region_manager
+            .allocate(serialized_len, !writer.is_skippable)
+            .await
+        {
+            Some(slice) => slice,
+            // Only reachable when writer is skippable.
+            None => return Ok(false),
         };
 
         write_entry(slice.as_mut(), key, &value);
@@ -785,76 +761,48 @@ where
     }
 }
 
+const ENTRY_MAGIC: u32 = 0x97_00_00_00;
+const ENTRY_MAGIC_MASK: u32 = 0xFF_00_00_00;
+
 #[derive(Debug)]
-struct EntryFooter {
+struct EntryHeader {
     key_len: u32,
     value_len: u32,
-    padding: u32,
     checksum: u64,
 }
 
-impl EntryFooter {
+impl EntryHeader {
     fn serialized_len() -> usize {
-        4 + 4 + 4 + 8
+        4 + 4 + 8
     }
 
     fn write(&self, mut buf: &mut [u8]) {
-        buf.put_u32(self.key_len);
+        buf.put_u32(self.key_len | ENTRY_MAGIC);
         buf.put_u32(self.value_len);
-        buf.put_u32(self.padding);
         buf.put_u64(self.checksum);
     }
 
-    #[allow(dead_code)]
-    fn read(mut buf: &[u8]) -> Self {
-        let key_len = buf.get_u32();
+    fn read(mut buf: &[u8]) -> Option<Self> {
+        let head = buf.get_u32();
+        let magic = head & ENTRY_MAGIC_MASK;
+
+        if magic != ENTRY_MAGIC {
+            return None;
+        }
+
+        let key_len = head ^ ENTRY_MAGIC;
         let value_len = buf.get_u32();
-        let padding = buf.get_u32();
         let checksum = buf.get_u64();
 
-        Self {
+        Some(Self {
             key_len,
             value_len,
-            padding,
             checksum,
-        }
+        })
     }
 }
 
-#[derive(Debug)]
-struct RegionFooter {
-    /// magic number to decide a valid region
-    magic: u64,
-
-    /// padding from the end of the last entry footer to the end of region
-    padding: u64,
-}
-
-impl RegionFooter {
-    fn write(&self, buf: &mut [u8]) {
-        let mut offset = buf.len();
-
-        offset -= 8;
-        (&mut buf[offset..offset + 8]).put_u64(self.magic);
-
-        offset -= 8;
-        (&mut buf[offset..offset + 8]).put_u64(self.padding);
-    }
-
-    fn read(buf: &[u8]) -> Self {
-        let mut offset = buf.len();
-
-        offset -= 8;
-        let magic = (&buf[offset..offset + 8]).get_u64();
-
-        offset -= 8;
-        let padding = (&buf[offset..offset + 8]).get_u64();
-
-        Self { magic, padding }
-    }
-}
-
-/// | value | key | <padding> | footer |
+/// | header | value | key | <padding> |
 ///
 /// # Safety
 ///
@@ -864,29 +812,22 @@ where
     K: Key,
     V: Value,
 {
-    let mut offset = 0;
+    let mut offset = EntryHeader::serialized_len();
     value.write(&mut buf[offset..offset + value.serialized_len()]);
     offset += value.serialized_len();
     key.write(&mut buf[offset..offset + key.serialized_len()]);
     offset += key.serialized_len();
+    let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
 
-    let checksum = checksum(&buf[..offset]);
-    let padding = buf.len() as u32
-        - key.serialized_len() as u32
-        - value.serialized_len() as u32
-        - EntryFooter::serialized_len() as u32;
-
-    let footer = EntryFooter {
+    let header = EntryHeader {
         key_len: key.serialized_len() as u32,
         value_len: value.serialized_len() as u32,
-        padding,
         checksum,
     };
-    offset = buf.len() - EntryFooter::serialized_len();
-    footer.write(&mut buf[offset..]);
+    header.write(&mut buf[..EntryHeader::serialized_len()]);
 }
 
-/// | value | key | <padding> | footer |
+/// | header | value | key | <padding> |
 ///
 /// # Safety
 ///
@@ -896,24 +837,20 @@ where
     K: Key,
     V: Value,
 {
-    let mut offset = buf.len();
-
-    offset -= EntryFooter::serialized_len();
-    let footer = EntryFooter::read(&buf[offset..]);
+    let header = EntryHeader::read(buf)?;
 
-    offset = 0;
-    let value = V::read(&buf[offset..offset + footer.value_len as usize]);
+    let mut offset = EntryHeader::serialized_len();
+    let value = V::read(&buf[offset..offset + header.value_len as usize]);
+    offset += header.value_len as usize;
+    let key = K::read(&buf[offset..offset + header.key_len as usize]);
+    offset += header.key_len as usize;
 
-    offset += footer.value_len as usize;
-    let key = K::read(&buf[offset..offset + footer.key_len as usize]);
-
-    offset += footer.key_len as usize;
-    let checksum = checksum(&buf[..offset]);
-    if checksum != footer.checksum {
+    let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
+    if checksum != header.checksum {
         tracing::warn!(
             "checksum mismatch, checksum: {}, expected: {}",
             checksum,
-            footer.checksum
+            header.checksum,
         );
         return None;
     }
@@ -947,56 +884,67 @@ where
     D: Device,
 {
     pub async fn open(region: Region<D>) -> Result<Option<Self>> {
-        let region_size = region.device().region_size();
         let align = region.device().align();
 
-        let slice = match region.load(region_size - align..region_size, 0).await? {
+        let slice = match region.load(..align, 0).await? {
             Some(slice) => slice,
             None => return Ok(None),
         };
 
-        let footer = RegionFooter::read(slice.as_ref());
+        let header = RegionHeader::read(slice.as_ref());
         drop(slice);
 
-        if footer.magic != REGION_MAGIC {
+        if header.magic != REGION_MAGIC {
             return Ok(None);
         }
-        let cursor = region_size - footer.padding as usize;
+
         Ok(Some(Self {
             region,
-            cursor,
+            cursor: align,
             _marker: PhantomData,
         }))
     }
 
     pub async fn next(&mut self) -> Result<Option<Index<K>>> {
-        if self.cursor == 0 {
+        let region_size = self.region.device().region_size();
+        let align = self.region.device().align();
+
+        if self.cursor + align >= region_size {
             return Ok(None);
         }
 
-        let align = self.region.device().align();
-
         let slice = self
             .region
-            .load(self.cursor - align..self.cursor, 0)
+            .load(self.cursor..self.cursor + align, 0)
             .await?
             .unwrap();
 
-        let footer =
-            EntryFooter::read(&slice.as_ref()[align - EntryFooter::serialized_len()..align]);
-        let entry_len = (footer.value_len + footer.key_len + footer.padding) as usize
-            + EntryFooter::serialized_len();
+        let Some(header) = EntryHeader::read(slice.as_ref()) else {
+            return Ok(None)
+        };
+
+        let entry_len = bits::align_up(
+            align,
+            (header.value_len + header.key_len) as usize + EntryHeader::serialized_len(),
+        );
+
+        let abs_start = self.cursor + EntryHeader::serialized_len() + header.value_len as usize;
+        let abs_end = self.cursor
+            + EntryHeader::serialized_len()
+            + (header.key_len + header.value_len) as usize;
+
+        if abs_start >= abs_end || abs_end > region_size {
+            // Double check wrong entry.
+            return Ok(None);
+        }
 
-        let abs_start = self.cursor - entry_len + footer.value_len as usize;
-        let abs_end = self.cursor - entry_len + (footer.value_len + footer.key_len) as usize;
         let align_start = bits::align_down(align, abs_start);
         let align_end = bits::align_up(align, abs_end);
 
         let key = if align_start == self.cursor - align && align_end == self.cursor {
-            // key and foooter in the same block, read directly from slice
-            let rel_start =
-                align - EntryFooter::serialized_len() - (footer.padding + footer.key_len) as usize;
-            let rel_end = align - EntryFooter::serialized_len() - footer.padding as usize;
+            // header and key are in the same block, read directly from slice
+            let rel_start = EntryHeader::serialized_len() + header.value_len as usize;
+            let rel_end = rel_start + header.key_len as usize;
             let key = K::read(&slice.as_ref()[rel_start..rel_end]);
             drop(slice);
             key
@@ -1011,17 +959,19 @@ where
             key
         };
 
-        self.cursor -= entry_len;
-
-        Ok(Some(Index {
+        let index = Index {
             key,
             region: self.region.id(),
             version: 0,
             offset: self.cursor as u32,
             len: entry_len as u32,
-            key_len: footer.key_len,
-            value_len: footer.value_len,
-        }))
+            key_len: header.key_len,
+            value_len: header.value_len,
+        };
+
+        self.cursor += entry_len;
+
+        Ok(Some(index))
     }
 
     pub async fn next_kv(&mut self) -> Result<Option<(K, V)>> {
@@ -1164,7 +1114,7 @@ pub mod tests {
     }
 
     #[tokio::test]
-    #[allow(clippy::identity_op)]
+    #[expect(clippy::identity_op)]
     async fn test_recovery() {
         const KB: usize = 1024;
         const MB: usize = 1024 * 1024;
@@ -1206,7 +1156,7 @@ pub mod tests {
         // [3, 4, 5]
         // [6, 7, 8]
         // [9, 10, 11]
-        for i in 0..12 {
+        for i in 0..20 {
             store.insert(i, vec![i as u8; 1 * MB]).await.unwrap();
         }
 
@@ -1214,7 +1164,7 @@ pub mod tests {
 
         let remains = recorder.remains();
 
-        for i in 0..12 {
+        for i in 0..20 {
             if remains.contains(&i) {
                 assert_eq!(
                     store.lookup(&i).await.unwrap().unwrap(),

From 10fd0f6d2ac937a3cc427adaa3dd8815e4017a4e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 8 Sep 2023 17:51:44 +0800
Subject: [PATCH 098/261] chore: monitor env setup (#127)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .gitignore                              |   4 +-
 Makefile                                |   6 +--
 docker-compose.yaml                     |  58 ++++++++++++++++++++++++
 etc/grafana.ini                         |   4 ++
 etc/grafana/alerting/1/__default__.tmpl |  53 ++++++++++++++++++++++
 etc/grafana/grafana.db                  | Bin 0 -> 1040384 bytes
 etc/prometheus.yml                      |  16 +++++++
 foyer-storage-bench/Cargo.toml          |   3 ++
 foyer-storage-bench/src/export.rs       |  53 ++++++++++++++++++++++
 foyer-storage-bench/src/main.rs         |   4 ++
 foyer-storage/src/metrics.rs            |  10 ++--
 foyer-workspace-hack/Cargo.toml         |   3 +-
 scripts/jaeger.sh                       |  20 --------
 scripts/monitor.sh                      |  17 +++++++
 14 files changed, 223 insertions(+), 28 deletions(-)
 create mode 100644 docker-compose.yaml
 create mode 100644 etc/grafana.ini
 create mode 100644 etc/grafana/alerting/1/__default__.tmpl
 create mode 100644 etc/grafana/grafana.db
 create mode 100644 etc/prometheus.yml
 create mode 100644 foyer-storage-bench/src/export.rs
 delete mode 100755 scripts/jaeger.sh
 create mode 100755 scripts/monitor.sh

diff --git a/.gitignore b/.gitignore
index bfefebf6..804ff444 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,4 +3,6 @@
 /target
 
 Cargo.lock
-lcov.info
\ No newline at end of file
+lcov.info
+
+docker-compose.override.yaml
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 0c9ba321..5ffae04d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: proto check test deps
+.PHONY: proto check test deps monitor
 
 deps:
 	./scripts/install-deps.sh
@@ -19,5 +19,5 @@ test:
 	RUST_BACKTRACE=1 cargo nextest run --all
 	RUST_BACKTRACE=1 cargo test --doc
 	
-jaeger:
-	./scripts/jaeger.sh
\ No newline at end of file
+monitor:
+	./scripts/monitor.sh
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100644
index 00000000..d4d1bc7e
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,58 @@
+version: "3"
+
+services:
+  node-exporter: 
+    image: prom/node-exporter:latest
+    container_name: node-exporter
+    restart: unless-stopped
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+    ports:
+      - 9100:9100
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    restart: unless-stopped
+    volumes:
+      - ./etc/prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    ports:
+      - 9090:9090
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+  
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    volumes:
+      - ./etc/grafana:/var/lib/grafana
+      - ./etc/grafana.ini:/etc/grafana/grafana.ini
+    ports:
+      - 3000:3000
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+  
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    container_name: jaeger
+    command:
+      - '--collector.otlp.enabled=true'
+    ports:
+      - 6831:6831/udp
+      - 6832:6832/udp
+      - 16686:16686
+      - 4317:4317
+    
diff --git a/etc/grafana.ini b/etc/grafana.ini
new file mode 100644
index 00000000..8c64ef71
--- /dev/null
+++ b/etc/grafana.ini
@@ -0,0 +1,4 @@
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Admin
\ No newline at end of file
diff --git a/etc/grafana/alerting/1/__default__.tmpl b/etc/grafana/alerting/1/__default__.tmpl
new file mode 100644
index 00000000..b8633d16
--- /dev/null
+++ b/etc/grafana/alerting/1/__default__.tmpl
@@ -0,0 +1,53 @@
+
+{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ if gt (.Alerts.Resolved | len) 0 }}, RESOLVED:{{ .Alerts.Resolved | len }}{{ end }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
+
+{{ define "__text_values_list" }}{{ if len .Values }}{{ $first := true }}{{ range $refID, $value := .Values -}}
+{{ if $first }}{{ $first = false }}{{ else }}, {{ end }}{{ $refID }}={{ $value }}{{ end -}}
+{{ else }}[no value]{{ end }}{{ end }}
+
+{{ define "__text_alert_list" }}{{ range . }}
+Value: {{ template "__text_values_list" . }}
+Labels:
+{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}Annotations:
+{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}{{ if gt (len .GeneratorURL) 0 }}Source: {{ .GeneratorURL }}
+{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: {{ .SilenceURL }}
+{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: {{ .DashboardURL }}
+{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: {{ .PanelURL }}
+{{ end }}{{ end }}{{ end }}
+
+{{ define "default.title" }}{{ template "__subject" . }}{{ end }}
+
+{{ define "default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing**
+{{ template "__text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}
+
+{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**
+{{ template "__text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }}
+
+
+{{ define "__teams_text_alert_list" }}{{ range . }}
+Value: {{ template "__text_values_list" . }}
+Labels:
+{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}
+Annotations:
+{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}
+{{ if gt (len .GeneratorURL) 0 }}Source: [{{ .GeneratorURL }}]({{ .GeneratorURL }})
+
+{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: [{{ .SilenceURL }}]({{ .SilenceURL }})
+
+{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: [{{ .DashboardURL }}]({{ .DashboardURL }})
+
+{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: [{{ .PanelURL }}]({{ .PanelURL }})
+
+{{ end }}
+{{ end }}{{ end }}
+
+
+{{ define "teams.default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing**
+{{ template "__teams_text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}
+
+{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**
+{{ template "__teams_text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }}
diff --git a/etc/grafana/grafana.db b/etc/grafana/grafana.db
new file mode 100644
index 0000000000000000000000000000000000000000..deafa6c1cac348c755ab7e215c08275fc0670db3
GIT binary patch
literal 1040384
zcmeFa4UijGe%RTVuWrm2L{Su^<x*@7S4%>i5$A)$;gHMaLNiSY%lY770FjiI0(yX+
znHB*w*k}wnqPR@YNGsd%$GKdWN-nP0&WG)~IG4n}+&Oo4t{l5^cAe|&*cWeHSGF&S
zQ}$KG-gBJnR;;b~{;&IW_iNw*oWW9BiuzAUAn-o_@Av-i|33QTzq+!ZTbjIXm}S+H
zr-Y}3NJRLOEDJ){aX}FNEBsIYB0mgYdc0roZ-n{Y;wzN#O76405ZfR7HIn9cV;{!8
z8T+%?e~A52?BB%x@7TYJ{YLE9hIYGud*H3X@9(a3{ZZG0NLu*47RBv1xE2@V7mh~k
zxLzr0cT@A%b49hbxn`(lF{c&^M!jO?^r9d3{9IZ|W)wNKG_PEj^R3e7<>e(g@8`<P
zC-OmI`NYo4&x-M*M<eeYv(&Yc=BM+&JV&d^Bwt-n!Ugr`^<p0C&nTCbw0t$4T1=*I
z$XApbax$A)PAx&sMP(@?FD=9W*@cDvQ8~Y*nKj+01licgTB*K~mov)sa2`vyN*X7;
zs9Gu~m}#Cb99v7l)Kp6Y$>#xICbg)91guw~Sbl6`{g4=+J`%A-zlRKSBUi6!ru(56
zz2EWX&hetWq;AJ}p`>4&f&gj4?s-Ox&p}1gPDR-jCFiAczUW1-5HI);tuMQjTFLr4
z(I#Va?W8EEd^8ZFa8Qg-ffU`%&2<xdwLjCQwX|4A3%&HDu{1ofv+<M|KX@?m{<Ncj
z5DfnvpaoK;gAmB_$knZ)mU(KW7K^4<t4(;SMKzZOPc{S>bZfgIsAho%HH5vRS91lU
z*cj}LYO}F`HN8~QD;qhjtm>tPf;!jUjUmupurGo3T3Wg4Y7!B-Uh{v)nmg2w%eiS?
z1@c2lm(qwtey}e-aWwM4R~tW-gTVh4ZC8s6i`XtdJ*mS9PHYUcZi=f|ex~7emAGj+
zO`#Eq-M6BSdHi6&(N4cq|I1)lQ~mG`_0<!|GpelJYHK<}s>0Y?R>!F#n?_(wX%MTK
zOlGgAn=G<k*@Ahvm|NTS3=w0{X_<~hxtdn7m^h~JAU;RtgTNSjVrpf>(sE?jwHgAA
zl;nBkQZl=ck;lnY<6H7J$rA6Sd&KymLy_+loem}@nEc-t?p6}iVUM=$wUV)+L))A9
z?xQ!WYOQw1Fkw1#OQG~?Y6-Z9f=snitTlucjB-`2G}@nbTBz5cOUgv*My!On+o{lI
zeO)J$sGo+4Wo>GK&81--TQsO|Q!7<->m_xAm}|c>s%3#XK+2L@vp^fQN=^kZA#H<L
zXjM_KIV+&{<v>Evg)O}Rn(3~|<X4xM7nI~uP+Q{lzP@<kP~^Tpdpk3`!-jklsd6<>
z)<HVgCQ-d}vE~!6_wvehl!&D4ZC@@`Heyn}a^0ZB;5~s1g5?8!@s|!p>b}@S4~PMQ
zf1hqywwK6XQ4lljn4Lti9~KZW;{*!`-#)r?oBwl&7SAkZ{WxzLY?=$)QIaT%Fc}<*
z*aQB2<6v~!=H?~8dzx!#OcrMDvKjT*!)pEjF%;j^_iUfDvbU|L#rTmUk=^s&!o{(L
z1P3YjAgwQ0!Z^VlS`ktzyVz#u1*aLUB3Wb47~hV^T*B^)iSf&T`+~0~FirRiU}n#v
z7fHlZEfuF_Hn$&0G*Kn5d85H5hM-8;hld@@z4N@opCl*$OJVM`lup);Ud2szg%~=2
zf5h*W#MQ38_{o!z8?J1(OE6u6;kaGbDg~_;{@ERpxFPZc8A+p%GqDqii1FjcBfCkb
z%(A{=supaea-gW8A6=pH@}am6)h{T6=vmWr1v2wO>jh{l%s7p98Ti2n`W5>%`273!
zBz*pN_6zX&efCrEdD|vi^&4-)$K@6Hm`uXQkyG$-_z>BfABuedpThsn+ndLJIrfXO
z|2g)LV}C#P_hLU5`x~(ziT!|^oMd^(->!K`66QYWjSfw@!$T$}NG56^JILj&iLsdw
z+|(s!@2Ff)t!7rkE7eXkuH7hayN=k$dVw_v-UQ6-OBfpgLkIQ*oQ{1=R)4`9&Is)<
z`kG@C=O@P}r^d!-$Hvdf<1^>RC(cdJ?0oSVNziuhL8oV(oxjYP;R{BoUarWNAy?|9
z5?LiA7cy|<;o5dbL6Y<H^4#)5c5z9zU{eJ;(|(O`zCU}Fi0G=<s*>SmyTFm=e1B8r
z^XcWQUhU1|b26lrr6intv=KP7%&VFM&>lW03D@id08AE(oJ?{+kC1|OsXw0RI)q`D
zZx+AVegrEmnIq6UPfNlTd&rZAoTWLCUvVb%Q$&VWobf_OGPC3$am1YJ6?V-*Lditb
zupr=RlPw&Ogp_^Qt2|g$?wi71mZQApUwBFsX1v}g>$MsjB)CdOj?2hKPxh@jhh9BK
z=ylTuu#4vXwSdjpu-xWJ{pzqNTs#T2(m_a=Q=?nhgo$0V#>N~|n^sn`skAcB5#8<^
zl7y@epQ;_RLm&e<DEJ|Y$mH9v1}+VX!fXl}GjAGI*(eohOC0suns7#|YdT>5@a7I0
znddZYXKO$bF1~*NN-P-FZJBgru4dHDf(BC(Y1P(bYH3wTXUJ5$94zlWV@MV({yvfX
zfKldd;>ZtYt4O|hi$e0<Rd({tom<E^nES~;&f1=Q@ivd-JDs}7S8t{w-`zcDLS9uC
zl(|f>Cns?qu|&)<FkdS9QhIsOg_YZy;DN98i^Adqg55bXmYo%^Oiopu-g3yuV?dvp
zl6`OJcs8UKZCk0+F?*=g?Lg<)l?WRcFzz(CTaYAS`fK%|tt6AT?D5*?D=pc6u!`H^
zveMHO-U^XwVDn;gA8Q?7!@>t4pN6v$ZZE5%C@kNAK2B?75Hn`UGmS71h;B`=OAI<o
z(=~%<Us?>A)jm;3%|R7KV*6SR7}o~UnKwM1c;j%jw@sn`fZ!MEjsuQWt@Mb(92wkj
z!YJ1(I_%A;yhQawON2)<t}ESb>vQC6Z%6hyq{q7?;ndd-@rHD}O$@abo$OyeiQ}1>
zCb(wEpDMYr-jvB4D1IlKkryJ8klf7$UFI*<$g<QOukHFSl)OWu9F);=uMZpPukUK`
z`RltW`22<4)A0E-yU)SrkL{A(`(N8B!{-m~yaAs-uyYAM-`$yj&+^V;_{{H+ef%5t
zZTP%uUx!b{{t|qivroe3gnbY`U%m$)FXrImS#nG;Fa{s}hlYlKNr(?$iNLP@4`aVG
zycYWhv7d<jN$j2BE3uW>#n{Q%vDmZ2|8@BHhyT&=Zw~*9;rqisH2h=3q#FD~0!RP}
zAOR$R1dsp{Kmter2_S(_D}h7N?o*NADn32B=pX9oJ{kmf^~bvpM{62v^R3!b&vd^O
zS*z<M3$725%cxz$hq_;kR5jDN%wkle;b*&#MVZriEq0*$SOiX!pmf;47jE^&ib8Kx
zI@o<E;uJ9WboY_Stu45v(lE8&7oO^#i>f8fwBSC;%DTQ$r&stlb&_EF*;w}!FMeRS
z`$*Kg@1$?2`yl!8^kDZ%3GR1-d&l7#&_<;UH{v}t(0yETPLbdyC~|SzI?&&JJZLW5
z(p2h-{i^URG{U>{oPs$ZbtfdXQZeA7FuCc#Qa7HCbRX{uP79#i$p6>h*L_%0t9tI1
zw*7o>_pB5YR5gt)o!t8k7o<SxhJ!i=yStA{{_Q?uSNEYPIfObOk}4e>X1%15D*GX0
zB=$!F{D*%?00|%gB!C2v01`j~NB{{S0VIF~kiaLKz;iwElTqgeMd#y7aCD-UHjSF)
z?*G3f#J(lC58-^W+lkU60VIF~kN^@u0!RP}AOR$R1dsp{Kmq{)sXOi-2Mj@2Bz8=I
z|L_k9AOR$R1dsp{Kmter2_OL^fCP}hr<s8LM&ytn#ZL~|*%x0NibkcOsVQknl6aTX
zXD8OwiL=`2*_r9_NmZTHPOqO?A1|(*oheSv&Yqo|of)4VU!RyhbEc@RX={b?>9y&#
zi8IBS(}go<3KK<jYHE6Fa%N4Notimw=JdqGnd!pmf;xNV%$ha@CC<#K+WP9ojdUqf
zQj6Q`8F+!r4du&UZhjlbEH`MP;cacR?)Cpa5n_MxY3^cdI}$(wNB{{S0VIF~kN^@u
z0!RP}Ab}@{z|-CIA%Niie{%gFoc`e-5<mh-00|%gB!C2v01`j~NB{{S0VMDQ6Cl_B
z>HdEhzLDbq@-O}&0VIF~kN^@u0!RP}AOR$R1dsp{Kmtz?0kYvgB-|8+|6piz@D~Ob
z2EH}$w+AjpZ%BVA8GYaE`^ml|y}#SL)_bhy$9q24{UcpzNV@mGUGtD6%ze-scwZzw
z2^f#j;1_jlrk(pm--khmt{15jWv-{mw!Ss@;S-nmoydp%;S#6dUJ&EGvIedm>NQS;
zUfI$ua4fO5?fO9`ej>r~v~pQV%S+jX1>$bmktkOKrxQ#ZQ+N=cGb#-Z!X3xaTG_BP
z@KRj_*Tos-dd4sL5xd;{;xm$<?cRe<r>@7FJ<v0r&nxv(NnI;xtbbg`Q!?Yj-YQq2
z;pUcJ0Qn0Aqh7J(SC^L;l;l$2x&C~A_Ue2xqqyp|s$^tdwhMBzH0S%9DtA1J&n+)x
z7nk;yf%@{LS$H*rn_o`V%}?pf*uw`U;hODvoNvy=&?BTC;&#0`f3yAs6nU=X-+7vN
zupfdBO@d=z*8?%B_!Y<T$tmZnqkVKFQ!m7+`RlGWyZ%w=h+19>ys0}8`BCZ+o$FxP
z77j>4%6CBzjVbCiqA$gLQ}8nGxA_;I5``JBHyp<_l?_+Ph+`Tu)Meji3#U8lF+#7K
z(krPo3v^klfWMgiwSdjpFuZx4_Uf=GTs#>JLT5}_gq&R1RLz=Zjg85~>v&pO$)?iE
zJV$i9Z%7idKKz1Fs+TMBrdF!v)=TO}ZNhVM?fTJuh$1ri_S*$ZgQ75-g2v1fAJ|5z
z$T->tx7p5U^<Am6Zq!@&)ZW<|kc5lxAAk}IMs-_eJPg5<L|U~qnOa&^(ivigmjjKh
zS0SlZ%oBs^`$8u_Krx-?0i77f9Xbj1EGzl$Dm(e+jsz!{TM}^{?qn<&<*HiQcD~c8
zizM@AD)QY_E5%wqA+IV6%3KB{`}ibc96R%+k}su~7hPB_z}NakVUf59cgeJLXFf?y
z$s_R-UW>M^)ae-Fqx~V3x*aGt8!)cn+AT<uF#WZ9&{k#y`c?T#OGYBAxE-!{JWUB!
z56lXIHo%?xnQ&^Eo5CHn3@;UYoeu{)3Lg$~^)wvjsdqTE-5SjwR@??Tg6E*k`nnE-
zGDt(0OxC6b11nscHzXg8e{oe5h2<O2$7zkwV$70f8et$1-I`#R7<r-!d9b`o$wfu>
z>`RLwv)U&LsX3^kNNk@w`({@ZcWQMAIORJwkj}i}=sd@=ulBYn)E^M8VQnnbRU=NJ
zl^#);BZC{(m~y?M->z%yYQd59QI6|MciZ|LIosDqN5o!-^mvyfoch`!-jJsC;CPnv
z%rp^<NXFc>AJdO^eD1suk%Z)KF6c6UsRr%?H{9{suJ1z0J2c8c8DwAAWM*r(OT5r}
zp`g`Zv*_;w8HAc?8vZVh3AgmJ2Aj-sm0ciu@!V;O?Ejw=t_ZQeGW_k~2SXnY{p`@{
z;C~$aZwF5g{7m#8L{CZogES@nY~S;}|Fn0p=X<;VxGNL+_rkA1hUUL_$H=yY+3dTv
zifjXji6=JC`JxvcgGH$`n}vo{d1tEi0)c&=&J?_coZWeTR1z-i5?f5y7G6$jl(#0@
z^|P0_!!D+haXZUtcxXrO^$1zcF^30C@FN`sXUiwS+TRJt1d|An+nKn?6h-+ouerQb
zevgpOa8?K0uGl^BvLsBszZz5s1?;sKI_!Y93m<kN!$2lSAuz-yywOKj4K#SNA-JGh
z+YLeFJg0Ge@s3^vU!%eC4=vO=v@jbBSkp@-aNd~<t-!n-rsixl9?3%Gto=el5>)#l
z^hwj~49zfKp<0>?+>7ePmI^^~cqdh%9Il3tXrdyX=8xvtQuAD;v6ClYa^0B;l!lb@
za05;2>d&(y>}oo-m`vZ0uP8U<WHz&$T7sr5Dob$Y#q_@?9(UA+ByLW@ZNuBgCBcr7
zqk>fpcK2=v%IgL+!#nq^$<`+9{O>@=%Lc4Z%{zKcldC3ZfK@lO(zdMKt?H)suw#X@
zZ+#y2_XViRSr+=|q&af*bo&l?yxr#Sig_tHE5x(A&%PuH=ikqg{vcaqKOcXC>_?M+
zhpuScAi5gvY+#ASBVU|RLcTy3l4DWwm2(hEf}PyX*(W*MBq7w`lCRF_4ySUG08{_K
zGuSUbezMVBvrmxeLSDx__?#r%u%p2^38vYg2p1{4sb%Kb#$@iG^+QPy(k|g@`Svk#
zC=_WgfTO?6o|;>|YbVL-J5;$7%&U6l!N?xk?yAN^+fFseS!9H&rx9EM-hHv<8pkig
zwdN14pW6JU*1O@{+IQcTVPgG&7#O<7@hc5ZFSuI}KV65@s|Sm>7;QWC(pEMtdf6$N
zPI&B6$eRP1iIg$iR6CoO(j4;<m~L8TU`tx8t;?!j@@)+>`+8zBW~If^LL;T%c%7Ij
z@5C;Uj5du)#_fK)_oyh;?nCc~P8<BbB}Wea55|f(%k9wlgP$sN2;qk^c{vtnCij9U
zWS2n<=0t+5@`L<=`3W5%@DlMVc=L!TtXyeVfoERY6z^CO&M8Iivk#L#Fd%J;rz3(>
z!Og>>u)5ezdPfDFUSUsd!gSn$!zupqXJ9eU9PInl=?Z)Ar~8rTCE@1YXOiGjE!UWP
zgR;q>wV6w1?mb5g`X1+!hX}Cm*~E^-_W*#=2LD<$7l3CYn*h)V?wJ1Svy$*8f79%D
zY%rlbga9?RO#-;#-1_nP|K@`X3_$`&00|%gB!C2v01`j~NB{{S0VMFG5n!JGf6|(U
z?Lh)a00|%gB!C2v01`j~NB{{S0VIF~C;{C6qbeW)B!C2v01`j~NB{{S0VIF~kN^^R
z@(JMn|H<!TY#|ar0!RP}AOR$R1dsp{Kmter2_OO7|Dz5d0VIF~kN^@u0!RP}AOR$R
z1dsp{c=8F5_y0v=|6G9o@DB+f0VIF~kN^@u0!RP}AOR$R1dsp{cx(xLAu=RLM~9q0
zVQOko)hqI{x$#P~>xfxS5V`&ziT$|%|KT4JKmter2_OL^fCP{L5<mh-00|%gB=F=A
z80<b8^-l@h>;Jzl#D4wBX&|-)2_OL^fCP{L5<mh-00|%gB!C2vz>`d1G9n7mXnbla
zSuE=n&c%LCkbC|A+d}NyPcj*78xlYQNB{{S0VIF~kN^@u0!RP}Ac0RWf$r{T)ZPDo
zD8xSe^m4$KBLO6U1dsp{Kmter2_OL^fCP{L5_p0M^z=ldQF8zPVc~BJvHb8K4FBkG
zX6XMIyg%^mfj9gAYV>!bZ%F@6{8906pWSQqzR>f3^{jUPR`<=WKj_jUza2R%d;_w5
z41ZT%6NQzN(V?kqwWwN}TvThDYldnTbC$XxTk2X#lM9=wS<|esF?ne@Bd3*>Y$~nH
zH{b4X_VQ&>Nbw5WZ$b8Rmn5NJPk>y>Vo}yBMeVM<ZkUXCItgZegSc~PC7Ds=)Y80i
zUCyWGujfLoVSeewyu7?5=R@&%`2-UHNfPHdZL|+6l908hn?$RY>Kl3`*C1CrnGACS
zlIQ%44KfjV+$8zYe2|DMl{GI4%MYkb1*23iS7bj+2$Wi~G?Vs6(A>P7oS&EHmKU;%
zfW^<5mov)sjJ%XxSdcj^-kp<#WqYUz#YRQ5QsMq-$AKgV5S-?FG2AxSUX_HD9SJv@
zQ}te2>{SHHlTj`!>6T4?kd%ZsZ82PF!PH>v6mx6aA=L7iyfwL(+JpT0hYRo}jsSIM
z@OCahkpJNVymnC(t{nvcZp-{3^a|JyRkjSdtQGZo+0yP>^47$vvY^am<Z<3m%6w52
z&aXmAMboHKmGq0tZB5Llm#@m1<f{uXrZ36L_0(!+l^wHN6M2qO?(|=fg!8)>pxlB{
z-409yNJGTinoKRND(MU%z04FxhMT2ZB`x1SN`fub`9en^38UTQ1XE(pcTimty``Bo
z-KaR<YhVoW37NK@SFTzp81;&y_nnk2UrH}8GSUMs9t=DF1!71~5g3k=Qd^o`Q5bVb
z4G|IA(OEiJvrDO!tZz?SX5oxm-)y-Yc%A>?yd>PPVTue37Pi`MIjl@v<4&s!n{>u}
za?Qui{5eUOdT%z6z}G#{5sJXqJ~Hz)X&_Y5pC^XPvl>^^sl{aahI~c2At$q$<<t_C
zv#2bA35EQ`jIxU2XJaEBV$|$1d0fqG4tMmlIoKoO3C7mosj)l{vt=f=sDy;`6q^&v
z#aN#eg+du1rnQP%)*6Q~8F4;WP*T#D6Ul`Pm{^xBj5NC9$ivcNDGB0)hGh%IzJFE}
zYNT4ncDM_Ku!h2dfKhRnXJOteJJmaOg(}h>@^Hu1sToPQWnTjUR9K+YEBfuaM#q-l
zAhLu4t5|c_Q&fUIj4iI@4Wo>dO*A~H4D@g`PukdG-LiN_5|TTPS#&KykQK^u`)yBJ
zPpj@k^MfkcQNt>y(|`tj=IWjO!n7o)_Qm!+5Lnq98CZLO-;kpLJ3$_#0N`y5+-b5r
zd7jGWYN+4-SAs&QOkBejq-rk*m(b3YDM^@pF9{M*+wWJukKTTKZ5#{dqqlQ*-=rkm
zv=0K!MSa60dnL70lGB=6l$R@|ZTWRwy8|l#V({qK>{Vi6Wnzo#HCS}7DjC@{U_*65
zKJF$tE?;|1Nh^M6P?}QIEyFy{S>Ifm5QWRkCd)5=pPRMhxF}o*mqU&Bp7(nXPPLrF
zJ*k|QlT8t^`^~D}n9NmkFO-ZnLXK<>xGHWRc|{U7?3qBtgPFi9BuBo0@udZ^f;2C1
zT4ZV#%PkZPKwJ6n{{N5R2m`A?0!RP}AOR$R1dsp{Kmter2_OL^@LeR}-2V>`{zqcp
z5@O$i5Bx&{NB{{S0VIF~kN^@u0!RP}AOR$R1fC=UJ>62I3mVkpUjP3KA@&!s|NJC1
z5Zi(TkN^@u0!RP}AOR$R1dsp{Kmter2?)YKq_-zJ6zL-Sf8i;I$n)^|obwGokN^@u
z0!RP}AOR$R1dsp{Kmter2_S(dionpoHQ`{_17Y~v!~eaM?*GgFpBg;WcYEL;_DQ`z
z*?VDlaA>#tw+G%D{QmAr*B^B~i2S<nd&2VIdo2+RT<iJM_~oM!`-1bDgYdg+g4ba<
zuR8JH_3@dH>ZK>OxEQ|x!o(e6!Y^(Lgz@EgzWMDBeq1g{|Bx4*?7aM}7(aS6^4>A$
zbr*gK{+H)yHO+4q`VQaK1YP-a@#DuMJ4eGclh@(+wUZy8Z6U$O|K6PSLt=dTNW>QX
z9`fIBL0?<p{f;+xju+*<IHUO;82+m-?4D=D_#9L;?dzqVn|ul0zTpKQqV+w}H*gdV
zit#CsqPw}dt`uJF&$MZ+o3gpFG(54h@st=pcrfz*w4(+P4F4SncR7T76tB%8@7MC)
z$jH1J3|>HGYPH&g7e(JPMuR6Cf(yE}-4Il>;B7VyVejbGT)`+d20QPpGaCz7(@Q12
zvXRrus$Oa+sPngtZ480-f_(|J*V4*WR~?DS^_u@X*4&|fT+U7F_NX69UM)i-68XWt
z_{7o317B_YR1O0FSF~L%E-dm=Q9nJY!wF7o476^Ft5|-f;dYg{X*o@y5sBTmqK<j|
zV8GE%zf}LrU|3WA@DBCW6NvZzCGUNR+}GTNRE1GyR>!F#lUF7+Dix!dOlGgA-Ita!
z5qf1ycVEyMSTkQ_T8$%7uBKJ2;C+7Vo5_Os9K7U_c|D#drdBpAEk}l3qqNTZ)|l4>
zdd14~%B5s>0bY+uUJK~IdC!-Owtz{NcrV=}#t$8ee6Q$qFfqa8|GsdylAsQIv~91I
z40y$@*E)vpK6<mN)@pYQ6KMFamL-j^!P`U|f=snitTlucjB-`2G}@nbTBz5cOTb}s
zyH15R>+8D4_7Lq<Yf}qsE)DD0qCqn^wNf>=UQ#!Rxo+Lhkg}xKEYL=+l2ZXp@Tx`-
z0%D<6MZHGe&ddtkq%Wf~Bm`a9(hH!O?i)wtSC^L;l;l#NRf*U8`r?U0k^BDa?ab^>
zr;=|XRj%gAI!Nc*B&v6LS056u_wvehl!&D4ZC@@`Heyn}a^0ZB;5~s1g5?8!@s|!p
z>b}@S4~PMQf1hqywwK7C;)$7d%ub@%4+{vGae@VeZy(*c&Hp(>i)Y^2=f`=|VAEVc
zsT4(+3=T!?0e`-6Fgk5>^OD~^%{4S83p01wjC$;06LEkTitp)rzR%fE*w)ix{K%2W
z?s;$F;@CohgA{y_)<@UPUT}w2gk*2^ibtmzts+@t&ls4F$6Uhhi;3~efIHYT!8Bod
z^FZhop(Ns|mWtCdo7)d0ny8Z3ywUKxITQBbVaIasJn!%)$;tmxm^&?{lb36H6*t)x
zVqb$D3YUE1YFA(U<jKelSGL<Fm@dI^z)MXl1+5nT*&UL&A@T$nNu!W6u@i}iWWThV
zbjmC{FHMB^Bj-R-LqEDg<>f<h9jaeY25D{6asXuJh1LtuR+w=b?K0t~nWo{->x^Pr
zdRc=$D_2<??Zqd^`~T_ne`gHh2NFO6NB{{S0VIF~kN^@u0!RP}AOR%s<PjkI|ARtS
z==$@aUmkj6@J|Pe1OIy9jsAZU{UPZOq!+}W7ccevM9+BlPj;0eUx#F=!I{WgEhu!~
zJb5@`N8rqG=fNp4e)w?Y{j77aqE_|XEp40rGaf!np`jffqqIB4V=vTr=N)iP0+Mi!
zRvz`K9yyS?Th-yRCv#fko{Ym$CVSc)K819XI7jPnT2^dzv~Fr!#;wrFI%H(d%k8Jf
z#Q5^Dh<(vN!X<u~ay6rF!a=)>El1AiwRN>#vY5y*+CR*}F%#RI)jeBel8_@t+ApKd
zWdOH|e8TP-72`<|YQjGcXR>)E*YzS3dV)&Re6r6ZVzRcE2FeC8#-DeN{XRjrN!a2^
zxP1Z01-CD7noks-2_D@KI(M+R9w&W03H79tB4NBN#%Dp0SR1`!WJv5qncDSP;{B0?
z7(aF_@_p|0Vy0~7>q%OTe@hM9gicPX+u4=2*B`Czs(&e%F*+@55nS+c&U2g2f9odi
z{I|uGNph_2nKUp1?qNDuUbW_nLW3blFXYs^waHi|M$91Ix%$vhC==U4lE9$w%YVBD
zr|YEO8cOc?;t819JG!-*g93_hU0hd7ti7cz3+y5!uW5z4spWW*KrCDbFT%z15=;^P
z8M|weiRlQR(+9;Ss>j9ndAKZhkQo)8ms()5Zj_3e>0jVD&Knd>$-PnG`<^>NjHz2x
zK2iL9<Ap%K_?+X3&;NO>U2W89pF8fEn%wY&{l%BWxHcQHZ*dw{HH|Hu_>&|z*7((t
zTd7<_<h6)R_xX0hunC(BY~Yb&QxikQN&sGuTytfQeNK$O2@P1?PXmIY*h*gNNcMe+
zo!B{c4D9P{WEaK=xpeM%9t^dXANn~eo^Jzr{5xD`1vQ4Q%~7k;!nlEZRn4rZC1-)t
zY~fsou*^7X8m+{T6*jfPtz4~M_68r+nXpG*6ysSKZ#SJ@4c}!-*Ea4zA=(%o4l>2W
zTTpn!_(e4IWN32qYa<>G9T%;McV+kH(?n|-ouv+~bY$;xIi{#-$ofkzcaal4D#laC
zBland;=?VVq45=TFT{9v<A&CM9+}3R4NT|_q~Wocu*DZZ7U-@ijx3b5@6g$orS09g
zjky}+2=$F$bUyL+5ivdkf&@1<_|@g8gJnO!yG@tN!S}}8)D2!tV&ib*9d&-ubiL~T
zKHAQ*(0*<0I`+MDpK<P}`<}V5&G3tVp@n7W&=|B`|2)RdVZoZT@*+(Pje83mc*tf<
zk~{T#UD~)L2ox*0Qw&N9Y(bEr$+khdU^Z``Zc~LD#0sz;^j9Rj(wOy$D>W2ft*@2z
zLc5I4_5X<Q{Q~^=$FUDv?Cd|9;MfO3Bp$ba;q%d{sYpZ`l73(7dd3YBydX)+O3`&=
zTQi-Hdn0P4V!)lXu<EFdoVzzt(rebpxi{~Ptkw0Bm8y)Kn;0D-3ru&TOOlWk`N+8u
zaEQ6CR@9Nv5g-M>$HwH#Zjd}SHgbP-1e^oIj5~79GV9vt$fjQOzv~5~GG~+w6SA2b
zYwC&dQF(G=0zRgvN9BnGlu4#Qh)o94$zQ3Dk58VFlO@fxYLJA=;g8CZ`!`9g;6;<I
zcU`X))Y1)2HCN%*8*9<1Seql~#z%?#)y<4ylq|jK1dvutjsk(oEkN<+P4ab1TQcs9
zoLh%wFth>gL)1zn?%qhjD3#PI94Ppa8>U{oYB&&~0-y%@CQpx!+;s}yhEI@=)Nz(6
z2r6-AH)!fAIR8{Pw4h-o)O>&R9-ls}qskiGs#qr^+zwDD&zzo6XQsx^PR~w^oj!eL
zZEUu9W_E1m?3r0@ZT<Ar+1XjrFL2vqsW@j;*7Xh2o%F!M5gb}YI_2I-8Sb_PnW}0@
zvn*|_0Ou#L(IL4Da1XXoCTY~WdTkE&2pVh^-q1|LC*eepk`~=kh+lY+k_0cPYg!2k
zra!Ni)B@aXSpk(BflZVF!E36SEa@8+Hx&A*XsUNst!<)7q;CkIb-4L_xvG-;+JUs&
zmqi_lp%qpPCmB(>OTdW;ThcbPO3+!B*YC7vxAb?Y_@t96u>Z4`>Lt?GGy$nOMcP_5
zO3rAZ(x@iLLn5!~MWR^H?^Q#uSgZOwAn^3)2+=-m9!8a^uhq#gBhpaSCHbHLsIs<c
z+__5fL5&`X8mL=X2)OI002>FV+cr$24thJ1g!*Z3In9Q`tW7v2+k`DrjdJt*(^y|0
zxeqDIYPAaGKxIVD;Z|8kM}vHf3aD!}qg1yDcDVVzO0qcG?+o4z6Yf@pm@TzbhvrrQ
z8PQm(MkAmYLFCzSXMBR+Y|`5XlmvnpF!#Z@(Hu$OcX$xNNP$5#0fW*XL=%vR3?krc
zR0FF;dl#x!O9fKc2s!avGMru<fi}Xev(=Ki?P;WAA)u=m{aZ6EqS$U<EYJdp?SdM8
zb@55o03WC%q&7^_9+y2ys+t=b=|;fBwVU6;$R)PTA64Wr1E?PkMTr@O1^o%)q4<Pj
z*5LsN$C%`dYOYv^aB>r#^B~8YpUdS)x*X^l9Gj}8n~8+Hwk^B)MrETqD#LI!Rbqr;
zAXq9HoZcXCs9ZA4vP#-~@19fH{rj@>uS_cicf0rQ8C3|rch4#6{(UG5ZtVu+MkfPP
zTTekJg7V&nzVR&Jauw3iq2b(Q2lr@$W}pp38wlfx>foo^1`<<9raZ7SbhUy;@Zm<V
zRx<8<vMk{$NGzi{#3kXUX-U}Gf^e4F!l}n#K?vLWZ@cgR|4@j1i2naRnqv+u6$u~#
zB!C2v01`j~NB{{S0VIF~kid77K(wbj>c0O!68ld=>_5f+<hv;i){O*^01`j~NB{{S
z0VIF~kN^@u0!RP}bVdMPZFq^?Ixo|EwV{+}NYFrUREkHOn;^T$^Z#+-wh+5B{Egwa
zhlL?~@Ee0aICyE``}(({KQH}t@%O~1`aaY9<GqQVpX+(4`^UOJ+x0ggdFOxc9nVR^
zg%9e{p(*!~3z>P2S9b1&4?V7ux+IgA5i8eItC>~umdNk)(O&jN{*6ymkSEUCKHfrZ
z_w=4bqfcRwJT0Dp`?NjY+wo((iSv`=lT%~kvt#3D<?)$w;}hqm&pvod5|Z|dpeV^=
z5ndB~)p?o)Zr(LPW#|KG@J?pXZ%1pGH;Crt<ovumx4e*Dgje(K=|LEoBY-X3l!Rsb
zkS9T<sND@$Pu`&5KJL@*2`)bo=P9)yj?2LA{)!~5+Hnw}sKLVx8r-!EZ)J6*Z~}?}
zjM}`ipkx&JQhIsONhe=>O-U<30(ctI8BD1q`NS%uoy*7&;w1NpCU_tQ-jswDI}u0$
z&jId52)|VFVBn*Y#(F~%F5A6rdd0azyIqgCk7zbOQs;(qd!l|r5~l6vX|4WasWLn~
zP%pz%t~@;qJ+&KFO1}Wln!rO_JRQ7yT@-HIC*Vz^%Jd}rU~<j^rvta|z(XM0^XcWQ
zvik%u7|y_Gde3jR$d>1IdhMDdWOp1xZB(~g)GcpK$d(~*O{SJsm2`$YqqfY9#0CNl
zPigz6lYC(x7bd^St2$(=rTIXSAoyX~T#aV(?5EXVemyGKQM(Mt>HOq8v?_<a$An5`
zg>#;LherywCfn&>`|J&7qCG?JNq3$Fwa;cH;enm)uq~Z8V_z-c4!4~n8A({&T?s@F
zD6{ke(7x7(mkT`2@9@FbK!T6;DR?d)`}`{C>2Z4k0Hxv4#j-|c!9d6mCK#jq7{+*z
z{+>uKWRx^DAuWbbUQR1Z$wftOJ=B>}dA4QeKpJfL?!|}oZu^zsC!}iw&j{X%k>8`U
zE0S=>&H)p$-hl_t;N9l3YkWd$9`9{O&SXWGA2731Rv-C$4S0s;(p6Eod=ezjI&XCd
zR84k-luKG=!`g&LZpqk8D=XPlTAAm}Ik{y~xN^f$=`uWP7luHe`6o}9JJTF}FrWMO
z2j?+#P6c*sNfNHvMARn8!T6vx!E>s6hw40k-Sjp9`tbf}o<wuI{r(~mNDSCPO1h|!
zo#zy5paTKW>{X&p!Ecx$GcshkP&MHZx%CsrUmBk+9v{_<3A)@O#idr|C0JkZdTw(;
z6y|Cahr5;`#f9qRFK)u?8OZaHm5kkWMG~&qN5XZ{huimE-yR-{=Pjq~-Y>(n;%_Qw
zgZ0`a=M6pkmLgcaw|O|9D@R|7Y%RRa4VA-xbHKg@@pHC&*q<ked;kCM2(jOZeZBKd
zJ?2CLNB{{S0VIF~kN^@u0!RP}AOR$R1Rf=U)yVp|HaR|hc5Q5Ja&~g;^z=+&>};{P
z4lhqzE3VHJ&Ww*wO(%My**zufa`*q=6k^|uedAHeiDe-HB!C2v01`j~NB{{S0VIF~
zkN^@u0v{iNV~tM$JWT?k-R%AU!c!lgJXjMFKmter2_OL^fCP{L5<mh-00|%gB=GSQ
zAlLtK|NrrG#~P6U5<mh-00|%gB!C2v01`j~NB{|ZoCI+H|8dgBx{v@8Kmter2_OL^
zfCP{L5<mh-0113|2=vFkEkt795@O$i5Bx&{NB{{S0VIF~kN^@u0!RP}AOR$R1fC=U
zJ>62I3mVkpzW?v*0>1tKN$Or~3lcyANB{{S0VIF~kN^@u0!RP}d`bwM00#h3b@I&V
z33X;_?CkXH#MtT6XV$<;z?s>xnX_kRwYBxrQ)g#q-TnW!1-K31<fnuUHWLXT0VIF~
zkN^@u0!RP}AOR$R1dza|ia>XFG&%$={;V)B#O@CN%i-eCpA1z7#`^zWe{b~fMxT@Z
zmb5DVv1s-EW#9MoeWv%9dSC8&zx$1@Kj<<ezaQC-j0ism8R4&dPZZwNqC-<@t)iAS
z*;3a^nygm!+%0WewhZrUZfimwleZ?43mGLXXOgcjC~}@A%gbqHDY>Y~nPoo=(&s0~
zC#S~7XUE3R%HuQV#wX5ApDn*53Wb|cUC}hEvR)}_cjfGglA|T%)I!0iSFD^~%sJwa
zD(BP7SLM{wymDR6w@J&ZF1sxXSCrP(kv6ug0^)ha7Ve6|jn#0msrl=iRy38)PsXdn
zK5$18mhBm+MlBX8E6VAW22OKnC7Ds=>{4natL%l9yu9RhRbD=k56a7r<_Y<H;ykDO
zZf!}zjNQ!@30OT;48pUqY7)k+Y`ZQAihY1vSs*X1lj=tcyp44R+4)uPoUkNediMd;
zT`)D(VnzfRAGA()6r?W6OUoHqxt?0htg=x1^LjB4osm&4!;ra}PAw+WH{>hI4LO<3
zET@*BltpDJ<K$k-E-dtqLT+FGgPf#vFW%`IIip+;XLGnVg_=f5%Qq)>TLhxbgU)19
zFmBlz>eV8&mmACNC3~bM2^)5jc2m8g->z#^TcISiX4K7smNU!^GN|=huBff6^^(=x
zZzOfDRdQEtm?T7N-9kZQgJSvpX+L91!VP;0giOL%dWev9y-1~N9#D(~UYj6!d+A65
zF1u^DMPY0MgvvV5Wk@J*sinG>y-NB<W-60cl?;i$P&M_6wSMCGOXIV}<D+_!TX3l=
z2?_h4*IX#JV3g|RidwQXliK(^Q+i%b&d<wp%L~~>unSIuLfRzHE0>blg^WBdbF4z$
zkc2sVAZW+iHKURv#mjeeYg4Y&OQl+iCIv|^$j9&9KQ3Q;O-U<q5Ukgn{G4iQ6-l`4
z7^6jf!z6>WWNd8cl?}OABQsaYxTV9_78P6DRZWIENh+__Aj~P}IHyvxED0BFn0p;X
z0^3l`8I{tu(^dO|>}O>LIv4>SU%ey>bM*kPUePUGEy-J&S<{UQY3oFbR(eS;$P<)o
z5HikbUh0-8ES?OiVswP6O<_|tYnnAS=2(rivXV`um3dB4ci$#jLCiN9ay6|`H#MhU
z_s`1Bob#2!Gioh(*-@QT?TvO<^&M5rf$b}56+*AJx3NGJn9arX7B$Q5$3|`wfL;R#
z^JKwel!_E11C}hUj!inN9DhdfW|MF-?lR%X2C=_nnLwwiU{%HDrUuwsRZZ^4!nv~!
zSu!vsh754)C4=3&E(tdsJqXE1W@@H1YD~#`$rFv4(wh^uv$$N7_}&*aP>Sz45^!P)
zrJdqe?e_yavoq<T?VZf-biVS}1nj&^$7QpAk!1lfOrFv|d^G@z2(4D*q_3MLPH0uF
z)$SN3Ok%Fhr%RD~O*0#U2!S=V=8Hm8LXcj_sda0UFMSZ7;}puowvdFG-MQ#^c7afP
zv<sd4MNzm#tZ-<j#3%)^!^}_aj){p6?V6bI7CWbU0jeh(wT7J&n=@EFw%$FpNzXGc
zx7I{KeKTBb<IoMN<!*QQHS#L6Kc`B9W?zQt$d;#RnMPNKd*1!<Yj)Uh?MKj%w!0v^
zH!lf`_KQ?Z*WNO9G4i!v<udg-I~1mjhD{mQq>=r9B=))h|KT4JKmter2_OL^fCP{L
z5<mh-00|%gB(N_6_VLKGLNxN?i}Bf^p&@B#D0*gUN;)8Ad#a{U)~rpfUc2glgRr~X
zo0e5Qcj{CSe{ObscKqH5315U9Mseia$kpZ5%*g$QgYrqZCf*P<b?)@pd-wN#|Nox}
zu|L~a12HiYKmter2_OL^fCP{L5<mh-00|%gB=9&B819L_7!A%I+~@y46k;Df&O)PX
zNB{{S0VIF~kN^@u0!RP}AOR$R1du?71bTX+?(_e`QyrFo*^mGdKmter2_OL^fCP{L
z5<mh-00|(0PXqyS{h#jtC*T|TAODa55<mh-00|%gB!C2v01`j~NB{{SfhUXrdH)|?
z|9`?d6`O+ukN^@u0!RP}AOR$R1dsp{KmtghfdJY6KPRjS!@o29=Fq<#S{hs%92ofJ
zfpq_G^?xz?OVJte8{*e`f4X<D=gsc_rTbR*b6vM1|1R=_kYUe%J16Xe;PGwuKDaud
z-dk?h4e;yax(VExp#BZG{!*NjkIGXD<CFyah%tUfXoRJj8=B?$D`9-3l=QL=;hq~d
z*5%5h^$(J^p9T*dM}gGbrdrw1z!ykubInlAVos|R%<U?8RihrFh^LT6Wj>W%e05>@
z)m3FdnFH5aJcoOBzkNUwR_&LdSmH)0D9JC~^{K@5@8#8BtEko5rU4jOH(!1UfqZ$H
z2;_L1<2j<TkJ(Q}1-si9hq_kM?t)t}a4rVE-AbCJ<-pI3>kOq)0PxF2{O2{eUTRDg
z7L!xB*UixPJTb4@F>q=b@?zCk1y$fZJF{A_heaVrI@gswVCS(V4K8a%y<WCJV(?<r
zwp)+bLy}OoCxUV1dJ%D5t29dMxXl5zC+=0Ic`ij-W$?V5G{zI}BE%IB8A@E&BKDX)
zC<!(DN~3_p0cUgLy^aSzgshA__j!$?Fj?D4OEs&dvq<|ndw}?kIkiWDJCW4!vgmrj
zYHS4o+ETXNH|%~%xb`*TBavFsW`v$eod#cE?W}5Z&JIoNzFbK%_?sO{wsULcX&Ki5
zdqbOVp1oLNPP`G%7<L~7Cr-Ot)lCgtU~yesK3coKZbwC7H3xm`cxG~EMqjv`XCHD~
zr{!a=FI#`M47|KG=~Kf?fr;<joq1c5gq8OW??sJHKZZKw&4=IR4zv4gk$3}*14lBa
zf_pJj+cIv2rf1&+a?^a}J3VIS(x41Rm&x?h)^X-Lb{}!^>v@wbsWpq3`eIJC$aJz#
z8QAQs=C_z7?WgQsNhmmOtiYiocxW`Y^a6OgbzRWP>xQZ68x@&YN)V=^mt=ii9;c2b
zJ=us`Qks)--VBa4iN{*%a*`X*UGChsdnDnC@3^#$NTDSJw@aF9ZRi@#5UbeTqM(q~
zuCv&3tw;x5;oBd7+1a8K6uS#5WR@x(SJ#Z}d3}<xBcgC|83b?eGfB$>r*eD;lR-S!
zA={=c!1{m3S^pc=ZC@q9WhLmu)?{jFRY_-vaSvR7lev&i3B(JZGtZDOWPT+7I&O(s
z|9s@T=XRKUCyNsBck8T*S&zshuA|cAOX=lBaIWlnHE-cy&pz-#5_0zOz`6#GG`*aA
zcB0A2;I3|H1jDGEzwJ9;CH8^%+otP=W|7G7WSnL;FS$HX2>wPt*d&a*{@cDU30XVR
zSTXBf+hxJl%{c7zD)p8I4erKiFt_TR<L^qs`Q1A~-Lx=oh17O=6!3l24m$U>Uiw&E
zZx2cTtMIh&o5J9o!RMq``;xu?r1zPg-R{5A^{2vbcBNwVf&X{#oBdNm|7Y}{N0$bE
zboLk||MA&l5j*B+5$VgSsjX`!Y$~-Hb->DmyxP3eU}9-{SEZP6Mza%8;i=@|h~U&$
zyIs;@ZwPKe4d-{R22M)BeQ|Dre);0g)~py$BqHx0cdTEvR0nU)kj%2csdM<(`Oxwv
zACByBLG0-&YqN>Tp@oxEhmccj5~o$j`hY!;U{<JAy;5lv*UD#U8&}7qzK)l!K|6d$
zv%a_2ggtUjjISmlcGl?`vd0S(^<16%D8kWSpuN#-e8TCPilN#Q>S#uRMj7@AJ9bu#
zUx`QTlrN-n!sHc7cN#hOB*=-HZ7ElqoUTmX;gvU&jr#lL(L6nCO6&+TVmuy?>_VNO
z1x^;{<7`+vy|NyOMS_I{QH17w6^;0K55nx>Gh#e75wTA>oznjJqIpv^LqRb!M{^?Y
z^uz6#=6Iqo9*%_l!n7D)27v-CY0kl%IW$L}X{j{LX_<oBNyqG-6L$2p7*B$LW4;=5
z%7ux!In+D)X&z?HX><N`-yBDPIWzz6SEt1I#6;u+*Wfm%Yz}!PtlG^9IyAofnq3*5
z%_h0Yj58M5Q<Ep+G=fY&%?GdEBn9RG5(NF@pByu1yoo!LV*LE!i2ahUi{XWv<0_d3
zc_#|Z+Q-S`j)q_{7MM||qZ0$;!jGJUiX%-GgJRNbQ!NLgSD~-tOhvceA9&TV%HLae
ztg>3tOw0M0538;d#%QaV1QxJ5I_O(6W>w?aASP&z7RT5wCf3W~f?y&-*2?&j82?yT
zWcE8>6ywK_M|P*^5=-4K>2T@*E3<O7#QgYD7*;5bftFD&D{1*^I<=Tg-;l2;H{@hC
zvz%Ij+>6RmMrJR!1ZA{XFSYP;zonMyzRP)LFk8A+;)R1Nd*Zp?U&eV^5`$kC{%QQk
zk;u-)P~$zhF7lh)VQX6$G`#|sN}xsFF!h9kvD(sbCC0wc7vIM^`sRcf|A+<`*)P5|
z-WQ)Z9C_f66K^=W4WPeHH4l$)x>hD40aquF#t$Ekyq|Rku)AxgnaAhs>;=*3#rvPE
zUL=dQraeS(KA713+(j{d>{#S$F}Gd8){L<p$p#$_5_PDiEtQUl3rA?-i>-{d`rC-6
zaRxaz!BHM_nTRx-m(0@3+B-&tH;BQ_UYQsEoos+CO)ZDp<4%%yH0@Td1`B8Z`mr0%
z+oL6H>AXm^)825piSAwLKNrH?Lqnaeapo##%f(tt);<<hOZ6sa2DEdQ2uC+f029pf
z|9ifoj|q_g5<mh-00|%gB!C2v01`j~NB{{Sfln3z-2Z>F`U|B+0!RP}AOR$R1dsp{
zKmter2_OL^&<O$D|93(S^B@5vfCP{L5<mh-00|%gB!C2v0226Q5y1WbC#%0uS|or3
zkN^@u0!RP}AOR$R1dsp{Kmwf*!2N$G)G!YcKmter2_OL^fCP{L5<mh-00|(0PZj~<
z|4(@8lO;1siv*AW5<mh-00|%gB!C2v01`j~NB{}!j{si(-ybhbjRcSY5<mh-00|%g
zB!C2v01`j~NZ^x5fL#B7uIr)@TZs(}{qW$w9{lpahXV`!f8PJ!ME|k$Bhr_}Z;O9d
zT<HC^-ut~>J-a=<-QVARuIryc2K++;g7B_=QW7rPhoVDM$zoAvyo}0@Ygd{2|7>>T
z$|T*JpX2kB<C9Zk<FjMqXXWvkbK?`|rq6EMFH3@AA81*<>zcHE?VkJ6<s~`KNCh4@
zJ$JeZUfs)fLKL*4P_<eDf4pqlU-7+u%9bIQwW3}xTi|C~-kMld7L>V+JkBd|%RV6q
zi*`3{O~EMD%N2gPZk3Mr&<3HC^YilD@<MhITH}l1c?t%fq%sHNJ^Q#MWbNK?#eqvx
z=u1|Ny;UA$e@N}`+MgGN!ab-wTLu56noNDH$`pX(ySdgNL-y)?GNZ6k$*W4n%X(p4
zzV@1uR=nSGYE@o>#`CD(wO^8i<;KQ)GIY{%u8&wA_murPQCK_~w46~t+3noIrfSwS
zYi!JMgPm4ZvZ=H(&*{UR2lg>Zn0ucnr0d`|1jj9H%kzhxNxr(E$f-+Csmk@#YG$=T
zk52UGS(m%4!`asExxJy(<Y)Ft^90Pb2w8hX6fUkn2NX@C%EdpQUcM^(@Oj`lU7NLE
zd`P*I9hG~<mPKKORX^5y`+gPhl6_PZE^|uTZ$Uesv0s2mWtV{TG?@w}6Cd-mysgQQ
zqvS?iW}DKXUu(~)d)VJI<5u~jb~ev5SN5Cs5lOhPbCW}d4)Q&L>5$2zbV<)^Uf%vJ
z=wuceH&4cpQ7Y2z4m!hEffh<1>Uqv^dfh$@ih8AW6>g2~OV4u(wolrhk%Z|TGLKTl
zrQW;U65HwPN9^XChkC(&UKCbC1Gc`QS8_1jSbAlHa^Ld=WcS<8NrGk{gw~U_ptpP}
zIfYVJ?_@nxbwg-1weV^p6qlFT)eOz&JB@c7p_5siH?`W+c3cwf*w=!-a2KT66(tvz
zAm`3nUZ`G_$6%pGJ1e`CTFJspxiqg_m-BnboR^(-WJ5drrQ~RyRt}RZcM{pzvY#dM
z_)s8tSlox}QaF8w1HEY~2hja6jR)e(ZU->-P^+kGC9TNuGfy(=l}f8P-a?gBR?`Y~
zQ_HoAb_W^MN7jOw)S}`nXNXFW%ynlq$&KZbpR*5%LXsH|x9f(b?t2)Vw4ad#!zT8H
z8i7FZoDw~Kqg$h#rJ5U>)n4h^WONnINx*DbM)S1z{Ak{-fUj<N|37ZykpL1v0!RP}
zAOR$R1dsp{Kmter2|W1(@c#cNzmKtnNB{{S0VIF~kN^@u0!RP}AOR$R1n~Ml>Hrcz
z0!RP}AOR$R1dsp{Kmter2_S(dp8#I}fAaenTZjaZ01`j~NB{{S0VIF~kN^@u0!RR_
z|Dz5d0VIF~kN^@u0!RP}AOR$R1dsp{c=8G0{{PAEV{9Q3Kmter2_OL^fCP{L5<mh-
z00|%g-2bBvAOR$R1dsp{Kmter2_OL^fCP{L5_s|n;Pd}aejj5CkpL1v0!RP}AOR$R
z1dsp{Kmter3E=)8bpQz<0VIF~kN^@u0!RP}AOR$R1dza!PXO=#fAaenTZjaZ01`j~
zNB{{S0VIF~kN^@u0!V=D{|^bjAq=Gk|K;GVfo}|ax&LqUACCTj^goHeA->%AYrVTY
zANIW0{TJQWyMCr?z3U((eVqSxzGQz%60+~bqC-;!Q&TNXt{9fSt`}5GH!8Wejk;M;
zOR}Y|l{9&4VlJ&DGm4x^zPg~usY~+Gaz<9Jr&cqoa$YTIrj_Gm&&wzJ^LjBar<O9x
zWhE_NO{W%<=^OGD<%XQhW|mV+Q1GI%lyM4N$}TMQkIH$&+{m%HNM&A}TUo1ER|w5)
znZ2e}tek2E60v31>V<+<t8pR|=O@P}r^d!-$Hvdf<1^>RC(ccuy=q?+g*Q(E!ECii
z2sVSD{0f_@S<|esF{hbnWhI+REAyNp=j<<vLb9>Q{gz^%urEl0YOjK5YOyHmm7;c+
zC!Gsg{=8gK%Np(C>{4na3w^vauUwb&*%c+%+^n3NF-NlH<>e)w4nW^~9XXnJ5^^Eh
zFW6s@gcW-kWKDvs1*23iSLC8zBbt{pwVF}d(sE_!7j;8ZOO|F*92~vN%gOn9d2V?j
zy9hG1D5JT1<$2{&GP{tG$7N2R*nReSNw{E7g(U<{DuQ@r0@vJ^T%nwdG@$+MC9&*t
zk}z$@!-A}v+U>elDQxdc2tQYY06YLa_N*k#*wf*9^;)i|t*iBtwXceUY)s8`fVXe{
z&a3uWP?h&c`>D-pR?QAA)<<Be+`*Aiu7{1AwOxfV(-aKm!O|-mV0@aQ+_r)hod*+~
zNi8ZNV_A1>4>y)O;NG%lL_xU;42z~wrMO&<W*tVHrEcW5Cg#)2R|9Jww)9N0JWpL-
zw9iO_VymQ^jOsQuJYFuCOk~TDw<c3dt4caUriNwDet2n!?I3c1nJ4psD;@ku$g9eN
zGMACLvGS$#@}ig8D+=1e6|w;8EZAAcMp9*R&17Kl{35*i9X5JV9+Qcss#kOvcOm2I
z<#?qr$Lo_1v94ZDYHal)t&6p;UK&Ejs~|t>q2PA5y=tG9gw#$F`eM$V=G~lLSu&mA
zJ^9*9@LrY<hl($;zZhVrUfSEuar|cW+f$No$Bsa=(i$0Pv>lC&Ga8f+v;lf=Ya+Rj
zfz^oHSjsYJURqg7!XRr_%NDwH&z=;8+qb+CwGTC-<E&|}^<-(%2HtQRvnM3smZQk~
zRLk#@PQGZ5gHGDS1hv)4JuEssP>xPMN)6k#PZ1k-0Ay@fLCHn6wz+1gW)aL2B}h9k
zHGiG<7@LL}!er!zqPW^!uwM~{nL4NHngz?uR&_d&4VAfJ+*<1P7-?aYQ;okY1Y6gt
zHo5{N3vqw_N1Eq`gIOZxnzw3xf65+()x`H5go>%*bvn+oL=W%v4h_(jX4YV9)4b{+
zQ%tB=nynC|sHzpM)N)@zo6XJHVj)16w44c__GHsSpv@YAx13{kO`B?^2)i_X+iM!2
z(XDNsBzg%}mKw8zV~w_US1Z(=?V@Iy29(|;v9_g_a<Dg2y*RqV^TJ^z6fA0JTt&N+
za|n=(p#)5W+yt<%s%&Vy4I6Zrt6bcL5yb9y*@XB1KhDP)C>s($0!RP}AOR$R1dsp{
zKmter2_S(-Lttp&hlJrsLFoR=fs_5e(|;@aKSrO96vQ9y`-$$a#4^LbIk-CTtHQq*
z5(5K26gd}9BqBRw&e6JBfosz4HFY@S4*z<S9<F)k(4nLqo^mlYkh3}O!aI9x)B3Fa
z-Lx1_z8JBOIcM0;ab_SBJ@F&guYyXloby3$7J5wR$A!)cooix=-K~@upO}bz5O>Kn
z-B#cp@Uk}=e7S*OQ<e@Xc5s~cF<#e$<Ey=0MTLaCgF!fGgP1*ChK6(<Tt<gnb)tRl
zd7xKap>Arq#xqQaOxW_G7+;=<*o%(d1y|tMuJ-OZfP$O#V1??zer{fHB=pa&L-((Q
zl#`@OynpmnF&>XczVAq&n~oUs91ZMH>F|IW52%A1o!T5wlVMGc+kNBi45IRa7{7l3
z=ACwf$UQfwrraGzaIKlSZK`=}xrgz~`x#*mt%bR06dHJLU7_LU2?>{!7%}sSov&UI
z;}<SO-j4+%l2cZz_K+LTf;-yiHQ<oFaT?L#P;NJW_zepBkGZhHwvn5Fiq&+frLFa#
z%GC1CH&-U-YO33JUWV4CChViHiSd<0#9nDoIPaot<JD`X5?F#PJVc>dmBOlymxd^e
zk2Sa>!ajXjbe3Yr7!~pIgr&K{QwJ|;hf3f}#O=q{t1w*=<FoOI9Sc@r&bo%ZM(LED
zqC$6m`FVA|FMcu}sWsH(=zvqjYptrFYUt0Oqzl}z#M+z~p91;ChEl23q4W2pRwet%
zn4Tu3(=p{<S%yXZ?D=C6`=DP()zsEClU%K=<?1!(7EmT=&Q>yF+<>*Mv?9h+@b8qL
zGg!@fYYeaFf|v|PT7NCun62$v*DE`pc<-I7V*JGyBj0<`RZqXXpxRYhZPN~+LzUd8
zU4nUqTpXu3!xK)MtL8Kt=ZZOvft!rA4a3|HUrwiyEf&(2Zk2d9&RN=BxShAj9F{f9
z)C*pOCy}Xbc~eYN4*$Bdmx-~;>?Ld_SgRU^O$G}SWojGbKI+gO45SF|a0ISk0{r{^
zAPz$OYu4=49?Ih{3tFadgv!C)sm+1~B_-_0vKXHO>&@)-94q4)J>Pn}3hM=@IRk;8
zj~Tu>FE<&-!2qxmOJe-mg@}D67>i9iIC?LDm&^UPd7@ryC0BD&zh$h1&B?;D;`RSW
zbB=<gAOR$R1dsp{Kmter2_OL^fCP{L5_p^m;Qs${?j4j32_OL^fCP{L5<mh-00|%g
zB!C2vz@s66@Be=^9I+H6fCP{L5<mh-00|%gB!C2v01`j~k23+>|3A*XgR&t3B!C2v
z01`j~NB{{S0VIF~kN^^RGz9Se|3||SOF;ri00|%gB!C2v01`j~NB{{S0VMD^6Ttod
z<J>za8xlYQNB{{S0VIF~kN^@u0!RP}Ac03i0Pp{QG#s%MB!C2v01`j~NB{{S0VIF~
zkN^@u0*^BR=KBBA<18o2h6IoR5<mh-00|%gB!C2v01`j~NB{|RPJmqhe@?hB#GV=c
zsiCI^j|}|$z`6dPivE}A4@%#X{-O96VnK}d_V;|X`!BjTyMDH7H1cmEtHOVT43FZ^
zw6BZ8>YeD&RI*r<3r4A4uE>_UQIjo0Rx1_5QZ3yeL2D&VEm@lBewFh<WL{3r&&zYm
z3)#gbIZra@<&1JYBQIqa7UaqC$*HmN*|G7n^7zcT@riTOXWz1~iNdT7mCl<+RW7KO
zwqcmtIlU<Bm7;ccKD~TZPA$zV*X4X_{(3H`DrcA*kSxb!$#W{O`|Z~y;ii2!+%8kw
zfVPloI&GX^LLTJHD3_JA+sJw4QZl=ck;i$>ea+5_!j+RCOSW2swl;B<3!AE0)2y*E
zd1*N#r<Ij#Dy_`(8rEfJB;lf+fWlPhFb5Y~-f4MjYA&rLGfI1aTGc3Q=H+GRuz+q}
zK9P3;=STByio|)I6qEKUNU;-bCB@Vpr6{Vk%{4<ci=<DhYDFuNFAtZE%?F}-IUgeR
zu$?CClC7kk+@sX2^0gz#rsH!I>=h!>ORXfD*rP<&c2#TJnN9+}43d47$S~bXhVebh
zpq4b#g6?Z8ho6Wm%kDjUSrU>TJP)!IOpQBKwx+3iXOgcjfRbNwhKzDOwVGLN(B=L-
zk;WURSJSD*Wcr4DMY$m-vzg`85)`zkEP<JVF-J^Q|ETP#zn_bZ^Mwy$X|8%rGo7Y;
zm3hU56tyXvueVJxtm-sHy1ngMSVnY9w@RP|&EeWzixXVdEK@Id5wsyyQ`^ebz~=J`
zs%Upw#(`Z^y|NgY=uq~tcv0N$zh*Cqg0c*q?@S9usmNM3X9kkDPCJu>-}#)5=NYVn
z_M#+c&OjscT*au2)hqh#y5?C>M|sV9Nz1|fmGgR<V@df5snUlci8b{K%5xiRPuL5R
zux&3wliP>|ld!(7gG!Le%jukl3fLwKD{6DHhYPLPS43guC@~k#a=>ZgD=>Rmdc|Ef
zl(nK>FI%8?tIC2h2eTq~5;<spSrW?j5!!gV_NZxvx~b*-^7M-ALS$tPFGL*bY;<@*
z(omwOr7hMbn|2B&>sugHS}Pk{njF#-y8Ni9WsO~=WLFg0C9D|UVuhsUu2NpMUy}sg
zCK^w=KU7n5qrKwldXeg7b}6-z^(XrFLO`xoJs+C(Njko=#_Y>PT4J2(ItJ2uOU5m=
zRM%=SnmE0cy*f|UBh4+6SCxzmg<Pnbdc|5lar~w6+2Zk0y_k@%y{4oU5+|ot9plVX
zp?mfvNyys0VGSagxN_`EYoL^eXv&(cz$*4Ot$f{pl`$-ti8At5%OpIkmyjcvomwrG
zZNr|A3ig4WQ&11H;%m}**D%Pgg36z#v;oEE!P*h~;mlC`Q5@IOP>pvx4;E)kVXgGZ
zn<SXUq?a95XlwyoMcX}S&%xsT{V`~<w+d+0TskRC(02LL0-3h`si8$?Q2UQzh2YJ2
zu(n`Z9!%;?f>smnmS)z#_A}bTWU4!RiegR$^=g`Dp;i~YGqt%r>SbtMDz{tSwqFIq
zXh5eF$)?RM)@>4<n%*|*W<@Qvn4p_`fu`r0#!XnJ3wCrTywTxRw?~y{x05YJ@l33z
zu6xpUuNmy-$tz$l`p#Vce-t}<ECUH30VIF~kN^@u0!RP}AOR$R1dsp{cnk^P{{J!T
z6ch;wAOR$R1dsp{Kmter2_OL^fCP}hBPM|F|9iwhu{b1v1dsp{Kmter2_OL^fCP{L
z5<mivApyMp{}^@(ii8A^01`j~NB{{S0VIF~kN^@u0!ZKy6TtodBL<4aAps<S1dsp{
zKmter2_OL^fCP{L5_k*=;Qs$H>=YCU2_OL^fCP{L5<mh-00|%gB!C2vz#}Gr`~OD_
z6pKRwNB{{S0VIF~kN^@u0!RP}AOR%s7!qLi|JNQv@lYfrfCP{L5<mh-00|%gB!C2v
z01`j~NZ=6=7#he3&qRJv82HP`FZTbP{=?B9k$zSBy!dl{7khrC=S26duI<>*#4Zm1
ztD(O+^zz{SfxjxevDZomGLb_3`0>bl4=i=9q!rcL=9;0J#hh9wg?^Og(n>O;$eHA;
z3yPc%#pUG_{dv8Zms3j_<+75NuclLr$@C5RigH6vW;4sFCCI&~EM?@SW%xh4u+Tp$
z=M8fs$L3<=g7RCXsMj<zoShbHX=<4pQPs?{UaRRwC6LssQ=V5YC9?||`81KtG)h`t
z&M4OdVGE|FS{k%*9w=l|i%O^|^=h$IY{H(sCdLzq$j+F9SglkHOSM3XoTYArf4v!o
z9FFXeGE+z^lpseKnAb@O+bD?fLx&=}Nk?{qmi%7~%SnPdl&+$debQd74^fSddjiU>
z69j(0LV#9Q^%4`5u+OcDarIQhzO7e^+THAmk_&WkOEZc3Fd=#|M+2Ku6)2?aQfeiu
zz}T2quFLto7nqlqm)L<u>jBWr=!5o*=DnDN{k$s1S3vlSj_|4Z>x`&8;hThBqe4?@
zA6~*<rA+MH&5Q9<ry}nkZLl=0_^DwfYZA3XjSI?YH5{s{sZ}hmUz=@(V<EJfWtdtq
zw{9As`<~(R%n3>2^$It#y%N*d+}bw3=(1L;sT;h317=9|)Y%(PiJjXyF@E%D<oyXc
zifb0=1sEY{g!3gAM(u}ph`6j-re2uv<e+xh4V`QV-BL?+Ew`qF1*%v~YiTAox@gp4
za>~mv1ugi~ha0tCGE~UNiQ$%K>18daRgJ<XQ<nz{I?ahEwme6^wXpV1BIl!KspTp!
zRpRzrV*LDz5&NJ&5*b6RmNe4>lN$uR)jaNk$efqn9c)1au?2oy!ajOaj9-H~S9x{1
zv#s0qoK`8i--5EPx0QgOn<o#Q)kpIVWl|GZD3^G`e(ft_{Pv5Hoq;wIvIDfWxYza~
zZo8t|%Fbr=#BXRy?;B$LGQd97rYUq;5J0}U7peWl1hM}?Ueeh<TrW++?!EDFIjB7e
z%6fe-a`-l?jT~N@g#F5OG5%&EVrvc$GR+0mk(Cbn6J||!%@I!W6)IKpj2TYHEvm6U
zNe#~U2-aK9WJwo030NV|>}lm|M@wQnb)5FJ`8JH}W9+zg)|+Lmyawx0=eu5Hf^IdF
zW@5bzuJB9*k(?HmPrREJ<4I7pV{KH;+OBE=rEGJpL1c~&W|F0mT1d&91PS|PU5wv=
zCI>ys2<jK+uDWw{S<a+*yJfF3iCPrH<>C(l5721d+YBU(O))+Tvc`N=&6O}AdNU^`
zmI~EQh{Vp!h8RC~Eb`v8ze3Kr%D@z~NvoloCng4XKkBWMQ+lhljR&fOMTuu}_&Xr&
zuBDk371q-JS_r0am{GaAE{IBOu8Z-hLlOIsZ!jFmsdB<fo&4NruRKBGpp!x>Ocwl!
zTbdZ3hMJz|)<m`-q^k8jRpoA2h(#oMLUlP&iFC0qe)3SH7FIx4P?AM!p>kBKWlqrv
z;{WgY$c7MmefX~p4-GvS{5u0b-Tz<u|2jDdhz>}%#jlHB==<OLUh4g&-uu1Lp7n0K
z>tA$Ti2NhSf`5A_aLK+c3adw>LsO{=nb>NY46{W|E*PbHxguMJT+%8V)~0-VeD=($
zvY^am<ni;9<C9Zk<FjMqXXWvkbK?`|rq9|@yDAB)eF%z57K^f@3-Y>Q%HC>CHY##p
zk)aw`-!>~okYbNZK&~*owjl}+Zi6got!!**R2({X=rWBgiOC8_CL5Y2(dN_3S7Bf~
zJ6N(OWCY_Eljl*Hu`7~r-_ABs376rmP4~@+UyN%R0*OOoma|CZGF0tyRIuYaL#<?I
zSB#CT)=nGJuFSTrVl(oWuiQo+wM(RtcC>}6u*K%8YDFu-9>#5Kr)txtNS+Gawr`2T
zt(z_6b2_tAN!%?8M{Se#+oEtW0d+2t!DDJ0a0En*0hsuW5(F4`3Pw?r%ld}toa=GQ
zP;Ffl7T+XgsJArP*E-qS)Z`jW3bI;rL$g=O0V6vASCtHt;ez{}Q-}Smy(tOjIm64Z
zuC0X$j~>q+wAC}08N1a>@D%GEdqWbY?PJh<Vhw$9s9{x0uyk-uV0gcioS&EHmKU;%
zFx&c>m<0yixBF+e<1!CR++LT22lhdzea@)Bsf;CSclDYDy~kj#xq4I63Pz<^Bi&op
zOC_EDZqqCB1^I*{+{+W=<KqeW&ZcJ49+tl#&-#Bn(Vn$6Nyym~6ihHp(6ts=YDUC8
z%o4N@{iRx)otu}2+hd?hMZRQo!`G=~-hNSTBbk?mFIkUWkc6xqr_4N3<T!@(T43j0
zTzgV3k40$h`Vh71vDa8RJTBjn9A3-!A;&wmDhfK8+_TO#2zQ0SV5q_OqFmD~_uLy0
zY2~W5&DUt>X#9eF{J4DWH6;yZh#pkQUzEo=3jCIx7X^iw<DzL)=@1Tjz}e()O*)GQ
zck7e7BrDfbtC>}HpwnbLdvBk#bCRIgb*Q3XRJWOSkzDfDB<b_5snpV{lFkrp%Z(+I
z>7Ee@_5<W_);k4|=`yZ$68Tbkd9kt3+}0#l+&lJLFoWv$29ejfVni2Pdpr-3$7Et5
z>lOWW-CL(I!h_VWFhBic6#4b3mYWwQJ!v%a)6d>$;~#c(!n5)Ab^E3yq;{@A%jevd
zGgXnka@M%eS5#78A;_6IUn!vLI@ugE^|smu)9w$(nRf)tw>kaxS0v$%9f7u`HL}g)
zs(RS`P_2d@-I`1;WZ<eO-TgD-ff(fIdMOE)6Pq=+m6G+^Z;HY_G9;W)=gt~?S1>vt
znd?zuiTEnQ8U6Mb>^CH#?kIaFTKipB+0WZIKxG~I+p6s3K2?_C`Dhf7?Ei(Q_P&wD
zlt=&xAOR$R1dsp{Kmter2_OL^fCP}hCyxO0{QoCUc$63kAOR$R1dsp{Kmter2_OL^
zfCP{L64(;~-2d+h6DC3eNB{{S0VIF~kN^@u0!RP}AOR%si6(&i|4(%Hq3}om2_OL^
zfCP{L5<mh-00|%gB!C3=L;#=v-xDTGganWP5<mh-00|%gB!C2v01`j~NZ=Dqfb9RD
ziIj!mw+DY~aAIJk|Nn_*q7i9O{7La>-#_bnt@l@Y{#MUOccJSKyS5|$84}`O=LGDY
zonG?t{sDNwCV8zTefxjl$AY})lX!BVem>lHfrj2Q%*5tp_AS;yA<S#6!5Iek#nr-2
zk0ji*#jqTnqmA!C3a^NV%dolAEefe}g9OB9o>zT~)`y$j=w*-B+;?}nB;m5-GC}oS
zCNLmhalY%G>p{jD0h4HN{&cwSuRXsLk%XG_GEh=~=oP$d4fGY;R2t9UP0M$gK0K@Y
z_FX1j$Uy<E2j1fExo+XUqIP>nAn!>&06oR0=Za0mYyr<r(@q|4-8&=p14+2BdkZS3
zFFhtw%dQ(7S3%mnK{ed;PTu(I3GTa|@m}Xk-t<gHYOB|<dQ6*N1PnRdwshk1ldE8Y
z{Z&a=3|?H$zT?>ul4=&XtZA=JLAs!;LOSC(z2V7Fv+si&f(Ot~^kw0Y(z9=lJG~lt
zuR9Tjz8;)>g;!)}WJq|SYJy9>^%KXfm&Rv{$4B*Ig1#l3l#*I?9P{yNean7V6lV3X
z5g~bKT}`GlY&;+(<1>(FI_z%yo+K>WhpF~Ak6nt4yt;W$Y#-|7hu6-6{f;QikoPi&
zhu|x2?JdTh8_DfS?r5`b+uM?GYL~nKpB-(i3mNXawMk!g_(p9PhwC?$8Css`ykYlf
z2hUaet|)8}&pfWO1a3p9m(5p*KT9$vm9?T?2Pd_6EqQCAosXVf$-V>cYLC1}-ko1l
z!Lx_&6~ZrFc1i^`^+P<bpsrPeD%dyAH>K)7(H{*0_2OuMl*$X=!fr+(z+`=j!f%Q&
z_y+z!;Rkj83Ezlx{fU0Jl}t^qI5qhdzY^5B|1jzr0cx9DPA({`bIOUJ5E@D3lE)?z
zkm&evM=t6`+Ub>k=LPzHCf>-Y*ju8oLONm<{1o^}Whyx7*9q!aW1L#1AQ3T7Ucw8I
z&h=angzy{}*k1vc0)l<qt`U`LobLUyz21g)cz&_LY!QD7K>)4DgWS@*H05Ai^TM0>
zX)&%HXi7`Wdk_c)k#YLV18N_!EwXk@(&oAEh!0BegbqUdNrzb*1hF)INaVeW%C%y<
z&)79E+3#Nf0jU*hs?8r`lquJ$HRbBi#0BNFS}TOdE3<ZHEMjN~^ON`;y??7o4Z@>x
z3(QlT-KT9+5@z3D4*JFS2E3P*9Caw%VJD#D;C}9C>4n{_82df4r~R+74`bhq{aNfk
z#QrGuZ{48Ix4M`U2_OL^fCN|qKb`0v5F$}2G8Gy6%_Ci1Vb?bk;Nx@ZFYk>6PF`vw
z=kASwt2k@q+?)4C*6Mo6N>xVAO^l9^1$xb>n+1&|A?}n%&W(VVz;(5vj*O1L&K7=;
zjmekYAbD(T<o@Ugco+h2pd;rjv#yPfZ0be-yAICj=8TeILN;?_O+7I_Do;*Kz{m9T
zs63H?GRamAVw1kt>ze$P`uO<d897OOdDS2ZmBU~0jNHFTYAx!{7SgMCU9T0?(hW^D
zS1r}F7LAIvIdX1%l*nJ*%os+=(yLAYX~pCy5UAV&6mQ-nU$?X+<Ic#rb+uH3HaNZ_
zN!&d?k}CKd)xZ^EorvdBQ76xwo=|6|#?DU9PK=#CeP(TJws>ZCZ078lS#53o^wine
zSpsWa*Gfh3>$k3N5XMDqU9Fd(neZJBZV9P-BV};U2u!Nr&CIg2u|i3$)%5?Ly|)36
zBRTIo8v}qd0}#ZnRx4<=TCoXg1&#=igD(OgMXiRwkUJzk2w;W;xmwLMrW-&n=8N4u
zAix!+^vteoS;?J~i_gBtvG44&<BvSCtXv#_UCxf2b9_#8c5ElMFLsj0NhE(HcFwlE
zc_cljd#k^DrUw8a>@Ju5&vFMdT~%+r^{cnus_N;g%Cpq@s;ZTXERH}&0!~XMI*Cvc
zZ%azq(cK7OlIBb)Y-&kbvdgo=s#GAs*4OI=Aukn423-=WTA41d3QD3Nub0fL1W#0i
zyUW@Rt6m07qBv`G&fr2t$jjP}%~zC3ij`lfY{X(_aG5YMT?^7Woz&`RsM&R4(cY5p
zSz%N{xujCH&sPftVq!&L8K+oLE9HW5Vx^^wppZH&@FpF;OqER*Tq)B%pJn+T={r8y
z&n!ZkAxl@})v89vU@~nib7JX80A;Cel<!_==_p4Vgi0phS3=c<{3e~j$V99w<to`y
ze}eM2D$6J~3DY)cOubPqh^mFnsm}7+TK|2DQ4}f_lEcjFWp8{Y*b*kJs&s^}#%QaW
zRAEsa^PtQnsYD^h4Y2<6B|>$7&<!Sga04gfMq8m9Osta2B#9It-|^)iZ3~glh(w~~
zGk$HM(PfOEkDr6fu+5a30dpA)G`xliukk2(4e|wFd2OMsn~R#h;M<G-(L;RX{OA~5
z!GT=?d3Z%yV_gFIU*^v98}#Gi4M|ar-e`1^d3dvX6$NFJV$d0jcGo}$pR)eW>~pLE
z8yottGh112!HzyBv8D3ep>rE7DUAZyH|FG7oQj;^l-P*E`hJgf&4wD9tQlSUT6srO
zhNyom3+t@6F~>4%=cK%FZe(Ie5GT(Kjg3ykhgRd0@u8{Jb5oPUWHMim(%XNF(A!tT
z^!DdR=<Uzgr2oHk(A$4*@9O;q7w!FJj%NP<wfB#E|6}jJ?fsqJ-{}1pbO|pI009sH
z0T2KI5C8!X009sH0T2Lzhl)UFds|zm%Xp6(?-AoYY`h;a-aC!=4&%Mucn?t#I=Xtl
zPV@hN$o2ju{qFxCdw;L@w`lJFPkaA&x`Y=9fB*=900@8p2!H?xfB*=900@A<+ejeX
z(MHq#XqWjn5;gyZBj(>DVe@b25%X_Hr}?+N!~7d+Khn__Z40sKe`gf?|GkYBiL5~Y
z1V8`;KmY_l00ck)1V8`;K;SJRfb;*}BGAYN1V8`;KmY_l00ck)1V8`;KmY{ZMgo}s
zzm4#bH3)zJ2!H?xfB*=900@8p2!H?xyhQ{s|9^`>BO4F^0T2KI5C8!X009sH0T2KI
z5O^C2VEzBw2p?I400@8p2!H?xfB*=900@8p2!Oy_L;%14zeS*t4G4e$2!H?xfB*=9
z00@8p2!H?xyp04f|9>0dBWn-<0T2KI5C8!X009sH0T2KI5O|9SVE+FWfkrkU00JNY
z0w4eaAOHd&00JNY0wC}<5@759L%pBpdVh!B@B#r4009sH0T2KI5C8!X009sH0T6f?
z2n>X}xVAQqYkTIziEn;0loVyHtc=8mCx+tZhDPG<rw_H65u--L8?tm)3XI3*|L@^s
z?%1b${<vr8Xr=p)yTz_w?@C5v5heVK;j2eJ-uauIQpc~h|LgWdNDsZY?P-d9ke7a{
z_H2Yx^+dF1G_Od4Ch>Afl(zZQ%u2SZN=jC!Y8&QTu`E^#lKEK{`5}I5czP+BNGJK!
z{7iC%&+VBe$1lwDIY(@cKb14Yo04;b<~Lc)#V!!F(V_U%P<)b)Pn?erpC6ypKV5rY
zgqy0zTwR$co{8W!VYMJx>ZB8wW=W~b{QN?iPp+hv)64!`yK<BVX`N19NiOl%mr`?y
zrCa>9<SjmtNiU@4NzPnyK5e9*&&<wt4O;2F$(V3E&|3;Vom}xJH9}d^rVtZ3h0GAf
zbZRc?W*LZJt@ccqyRu0DM5SEe3*~jWlod2hDps_tR;*<2kU?oXGfNBCd53~cpjGT#
z5bKZB-W%bjYtJy&<;spfE2?>3EAv|;srlvPQks!m;B!9ttZu2uHo1~el#DNOB`b)c
zBB^RF#xEyllhbK_uZaBRrG>c$`JU1S>i=Y|FTzRs3D;I!yY?w+SUFRV8T3fAK?g@R
zE!h{wO3duhC~9Es>6O}hBHV>~)K$tS!Oh>PJFf-%;!<u0wPHiTW?S^sOmm=PsCJ{V
zt)HJsUQT3Y(|o)uxXgXEr=y(SuJ=%(mm~wU(N-HQip<GZK2QA`o+dGyrlx9gX|%MZ
z<a~mfPLmCKWCYpBX6@Z!PI!s*G8)r<>|@wySsr)i*V(E5pcd6xdn&@s85T(`z;d&`
z^k?k!Z>|--QhQf~OVsW*SRrruk-c51PwBVI5claZ)!LKP<n&%DtV>-J*3WE8JJ~Iv
zP_;Xa%zSDw<8)=2#blOZWt~`NgQrWZnbTG3tc=txx~em^CnDTX?J1WrCEc5>DgPi1
zo2k^?Y)us#t_Qc%(ONGX2;5}NK`KZz5D@YOUZEae<=4szZ(bC6QBXHl%Yq`RyigJ;
zs31y;IzyhE1_Fkoc4B0igqtrWmy&L(ER7tP??}z_r!4nRSN!Q>d}4mapX9_verPzL
zRn^{GI~L*A>qDfrDr`xl=9*9~XddO|l1CKN$Z$Vh-mc51aaZ(IS2S8y)=AYt`s4<h
z-z$<*lvPzOmsqG$E=cUJ<sjL|YDJ`Rl-L!GbwzI^W;4lUx_D}6c#!__!-E6nuy0^+
zz#SL*zr>gnJHg1{fH|J@$ML16i~`u`;aX3GOX+O9<l4Sl9Je6N{>F_(#IkNovl;k8
zS%dII2)3>#YDXj7vOe5G6_$xfJj;OYrH?0S3x&)W)JU1Q7Hu?C>yB{O^yhZbNt1*k
znZaKASkYSOVuuEk3Dvsx(t>ph`_RIQ)<O$AG+2wNT9mc^yv3qglJ8XQserGxv+g%p
z%=-B>yHGbWt6}$>o3w?t-l$ZIyCa;cMGm68AL|1tZ^Ja^Gp?>wNY)zM3p4^AH0L|?
zDVY3|wQz*X?AkKC^7b-CZ`}Q`4OFUjzjlO%2(PoIVa*C1xrS%mbGGF~XMMl{hX__1
zcIIjJkdw_RT={HX)i@}%XC2gNJb~6k&bB2tl@;J|Vs4l<$#aIeV@$T;RIFarO6&}f
zo5v+=@@RGm`<~~q=l?^!U*&qgPH%XD00@8p2!H?xfB*=900@8p2!H?xJURqA+uPc{
z8R}?ni+<Cb|NjNo`?cP`cytPgl7IjRfB*=900@8p2!H?xfB*=9z++F~NJkq-)BjHT
zgZcksZw&wd0w4eaAOHd&00JNY0w4eaAn>RX@Xi0t_5bI&W8Zh|$sY0OPanO|{R`b=
zT|d$F!RSAU{9!~s^2bMRcK%7{dpf>7^s}LpZJ*-a;Lda3!TmLk)2H;daNmg&q1pvw
zkw#v~Z%DNAMibmm+-ue<5^FiF8)B@~Fu%GB3cc&&{cA8ZA;({uUAW|~TBo%R(so6r
z_1&I@7oLUk)-nlNvmOv0qh-QexNrJ&NIz+;#`SL~pp`eJlDR2?EE+3tjD_{aWv<@5
zFs31jozh$t(-6p(?bwTYVtW6l!+p!7#iF4FTMOC{JzG(vEpsOXBm0J6u4ZY|yr`%l
zW}~2%$~9z|i|J2&D%_VMB`>yA(#6w|E2r;UKUYRe)jUyR_1XL3zSE~epP4hPuOZ)t
zuiRhr+hIfS0hbLKQhW=4Joe|&wP95(S#Bfa<`|}i>r#o#w|O&%f=mHfxe^fO=9^U|
zsT5%Rls^mcw+w4>ky6&!ic)`=r&PPPDiUlhKWdqtjG5Y2gM7+6rgwib+;`<<n?B_j
z8LdU!qUDH%a(**grX^E?Mk`^nRax`|e7dE1c_R5!o7UwCW-ZbupG<z~)o|a*lWnz4
z!?-+=Js&^iw<%A^0qx1JM02$h4Yn9KxJnCxs%52ZDPLun0^-D8xfkx6INqjr2Y5Wo
zl$~~Q-zuAD0|F<kg?sZmwkLMNeG8P?3jvuKChFhg#S#AMZW#w8Z15FK2~R9q6KQ;7
zYa(l(+z$60Ki>BG6~Du9vflRBr*r*)wP~c^)jL?5(KJ`It#fn8?u-A*)*d?fJnedt
zHTnzNLr@!|=4nMxq&4a{H+G3(_JG<>MG6#P^t`kUfV%|IC!a04WnHKzZS4Nta9?7e
zO+W4!gl}^P!>HJv9lopgnytYX?xZr!z!zrQqq$3peq$@#H!xtgXkSd<mmR+q_`<#g
zOL)_^LJ_oZ-`S_y^d84=`gW6`FL$jHH4hPdF-?T#z7YO>C-kry?t6;zs2VMYqR_uB
zKPNNYo4uce-mZlEl4L~#jupA<bJ@agcV(^bdbQb(eBqvSCI(+9vm{qA^$w9c)K`%f
zXIp6D*)_l?NZU&dPr?>@G;B9;uM+Nifi!uxg(lV#WS<u5URpTomf0R!zNCzV9U8k=
z-aT*20r~_ed*$tT&A`0v(3t+=Qn>FH6<l!4-j`3{4gkIw6-$%mR^f{lD5yo<xCzPa
zd$tnLKU{nOWrMJl52CCM>%q1T(4z$!?eAjT-cq`ssqvNes#`ZU>lOnm*cT;M%WWD?
z`?XISj>iq8+<adOR@T-u(-(F?r~R1h31B||{|49ljovpN)f*lt7YKj=2!H?xfB*=9
z00@8p2!H?xJf;LX+PSt+dpj4!{Qohv02CVpKmY_l00ck)1V8`;KmY_l;Bg?(F#rDw
z*ZY;;zkD1DjxvG(2!H?xfB*=900@8p2!H?xfWRY6;AltNGi~2Ak9Q4q)0gNs&BI~M
z-~a!LJ^%k#k8t@>G!Os*5C8!X009sH0T2KI5C8!Xcnk@!0{}zx{C_mc_W$FacnlGs
z$RGd$AOHd&00JNY0w4eaAOHd&@E8$bzyD+Y{}@#`iV6ZC00JNY0w4eaAOHd&00JQJ
zSQ5bR|Bq$0qtGA#0w4eaAOHd&00JNY0w4eaj~M~X{~xmoM{z*_1V8`;KmY_l00ck)
z1V8`;9!mmv{{OM8b`%-}KmY_l00ck)1V8`;KmY_l;4veB`Tt{9;V3Q$fB*=900@8p
z2!H?xfB*=9z+*{(J^%k6?tkWbf2ucr?8lD9dj5XThmQVu_fK`dKROrr@$i2O7moa=
zBhk+7j*qtggZ72cpSCSh<hSmnpQ#N;xKHbeXwRr1io9GBrEPlWg@UALSy5V(OR^@H
zOFa9htY>AB7fK?p$y!0;^W{RdSW>5#l8JPZPtDIHSNL3NW+mI0I%_7)8p(3}!aSdA
zjFRI|<;+;Q!JHW{c3~twGCCBW8j4Ty@rm>C;q&8@^=E4F2zTxEYoxIv<rPWLq-Je3
zVYMJxdZiPWW=Xrt{QN?iPp+hv)60R`cjaU;N2N$7uOyfF>r1J*#L_MPTJjd3$fOri
z^CWIAIiEJf%x7k2y9Rk%V<#2M!Va?vn@+Cy)8xw~(O{b6o5Dmv69O(*D-<A+b~6Mz
zotjJfl#olBq-+U=tST|ts#^$K9eyTxIgy!7^XKAB>n%x9sZMu^XGE(q9zGQvIq|N*
z%K1d?Y?!<95><^T6{MCGGqbdCop-9GWt9Zfey(;V!s(sInYk9I)VxvAyt+{)Ypp5;
zvIKV3G)&&K4l%pB!)rSgiPy^fs=Q9PmXou|=`<e?GB7<-8;Wo#{j{%1I60g#jdY0M
zSoW2fPc3E~^Uo|Mvrc5&?hTU6>YWguxwGIvp0#te!3g)lYfq9cmZ>`utfFrX?`Gys
zY|8Jj?Yr4Lm~B-AZKJ_z*k!jWnyt<oE@;8w1wqxwzNJ#u7lSp!KyrFd?R12@sSjAS
zSS`tSs*>g3@<M(?%Cfe0#{9S`?X;{+H*(gAY`QQvgw>`S7_87r?ZXl7LhY7Ym2P^o
zE_bU)H?C2WZf0f!^?ED2(KD>3Ke;r!aLMN}r0t5VNCA#w&#DM6TYD|WU_V7=OHePO
zNL%GiNu;V`O-|d8_^L{+QmAShbYW8}dE0J|g=JY-)(Fe-iJ2LGdSNy*M-9dm<*^H@
zbw0p3P|H0P;TH7gtvWY5vj*{JjPFk4^>kVd5wliAv#uKgS%ur-!AQ>31|nQhUk<Xr
z24QC^R7wwlmZsQ{F58f<rKXMXf;B!~`%r|->o4p_W4F5Yt*o1*rJkNBfhyLo)t--V
z$<K_Fik3TMNIZtMcSCAmctF2sNbwFAyp7IN41f2@T32ITG{-PCT$f7Z3Vpq=(Krh-
z1!Uz)K$x4a;VC^0#xp7j_MxPhQr6fQ!|z``32hJV^S_>OYm5?{KCn9Vr)wXKaLf7y
zUwfqrrSVC&P|k0jDOV&#V8hSsuHCJhBI`?G8k$ESYqtIv!HV6fos4i7>vVjVZD$@K
zd)t^N@&WD3uVk|sv>d@P8!`GyX<N!y8M**72rTqYt)Ir#dN-As`S67V^VX>u(_u3|
z?zEg>@9p>7Wz(O@qV3d8OK#=4+iNEx+{=28mAUD;?L2ADVGZbLV&Z6FIi5fV?ley}
z=5kW(>9luhe1yBA_xcbReI*Sb4+g+Y?H#zcKybfyob_G)yp0ag8M`_Kd4LKqBxKp>
z*LD~!^4HsIABb=n{b^s}6={ve30ALtPzIKe{aHd=?K$QX?4g8dhC)6ssp_|8il&Tx
zDe+?M{bBB=JIQdspD|o4*&|s)K`O0N3mKk@?>3vj@BeSz%?#uO0w4eaAOHd&00JNY
z0w4eaAOHey7XkMBf2jATxZeMVo&fl+w@VA;3j!bj0w4eaAOHd&00JNY0w4eak0*f>
zQ;`mOTw0Mdz9_Go4;j&eJ?bVee(g&q+dH`z+B&Rfn9Bu`FO=!AVP4QQsaVm}W8&BT
z<hwo?>g1-Q9r=PJl&Tfpe58_|j&~k%RK>6T@*lF_|3kfB<a+;o?-zT2<nd&UQiA{p
zfB*=900@8p2!H?xfB*=9z{5k}+2Gp&ozJzu(AM%!K>P9b>9$>N0fcb=-@{YG&;$fP
z00ck)1V8`;KmY_l00ck)1l}wHSpWZKk%e3k009sH0T2KI5C8!X009sH0T6h22w?vI
z@YDq~0Ra#I0T2KI5C8!X009sH0T2LzH;Vw~|8Evq$OQoq009sH0T2KI5C8!X009sH
zfrp0xTmRp7?8_Ye!wUpJ00ck)1V8`;KmY_l00ck)1m58UYJF`-*6WwsJA_iXG^9$Z
zDwj*=h1KDy)$!rAvGJ*qsqv9~er$DZZER#@YIRr=*5a$L_HW81vHyI(H&y>&zg(ef
z!|^l2V`FFNeI!oTs;Z=9g>|W<QP^Dho?IvhXUEUP`BQU3UM^{6b>jk`Drr)IrwjbT
zGQYwP$FsxZ*@+mRs8kBlO=<O-teqVnoj5akjz4wn#q`|lAYYI-CH{((-z>-Y=?$e^
zl+F$hpQCK)KYm$Q6BOA>LP`__Rm-Z9R3aTl;v=I&@u{KsWI8@|eq`i)eEiJR<k-Z>
z@MwJMcK`i>zrl%L`^rCNzyG)O{u)RB@B#r4009sH0T2KI5C8!X009sH0T6g+5Qw+0
zN4<9|_Pob|`TskkVxc7nfB*=900@8p2!H?xfB*=900?*pJn;M<=l^-=;Ufru00@8p
z2!H?xfB*=900@8p2t1+$u>ap9TCpe=2!H?xfB*=900@8p2!H?xfB*=12;ldB4-9++
z0T2KI5C8!X009sH0T2KI5CDNklt53%pK{0AVqC{hcT67pqhs>XpEz3X{_d{d3;%hz
zc;qKK-{|}(1<vlT@FeQ@!+q*-sP+u)n<#B(7L!?_ASqf_sTQQ{mZY$47tJqaWj!m4
z=I1JXumYQ-bKgC^luV?Pd}cnim`U=f`I+PjpF4o;9KSHn=bCfK@uzZzf~HQnLGv4F
zMISA_a<R{J>raIHhKEC+dqERc3sSS3<}2KHS-Phamu8bK^6tt}MpUeH@=9`vzrK{3
zODx^uuO)BsiA;JSHBTbulJjYPeu4ha%+7WV@;-H)q)ic63HWq!#ZAl4Brhj2vuQr=
zNGS<LNv#NZDcc;~iL|bitCa>>EU+RdwBw~+d2T6usz|C<RwOaIM!Q!AN3kIX7A3Dp
zf+kT(X2=TCskx+&geGeRDL}J)xg^ROnOi`ZC}={!W!j}vQnrKw?V!n0tHHXK%2}2^
zt7@cGgA@nvz<hazA`LJ`(zcY(N{X^aiUeyYa%tU2vsPA`b9bc2NvcpPl}Q6CAn9q=
zG1&$M6ogf&&~VMLZleyV&28@@d6)#qYPKTKcBUSqw5!wAAofYUJ=}NkL`Z+$Y3g|)
zzag1$%#WMWj(d&!Ov`rcM#?$~&Bp46utw?z#>Vu}k-jS@LY$GidZ!?3QdX#HW#hXg
z#;B)ZR|fYw`LXNaz7r=xwF^cgcEj9H+-v?u>xPgHT^=O}yFRY>hWjMKU}l_}S@C*2
z%gKp?y0Kao6p=MCf?JXbR1dDZ;QqFiqvhKkL`tJmW1Jj6=F6CznVoemrjP2!!hKm%
z{Ekz9u0mN$ai4F<8cofO#Qn{d73xOXMbfUC`*J4D&1N3cFX%ntKAF_MYp5Me%9q&e
zVhrPV)#Cn#{Sa>@=YceL+a6JG(~pMxULf^{cA-v2;?eYPJpg6HgZn!<pFObf*z0S0
zm(d13_jE%?=E?Ottz4TrE~5=Rc!y<~WT1mTL>FGIP@8YrSM2TJ9S3*L=pbA6OHMo^
znt^?ne%5D4wzqEV&0BS`@5KJb?{|FNljDM1Q!x82x1AhXAEi&|QS#2Qkk0n;mbP7Q
zA(yE8F436BGw#dgtBNt2F?x2hE!gq6|Gt&Kuw%5A<n<PIY}P70S(AhFWlo+Vtw+Lr
zcVnU2Q~OcfkVR&=YZ?Cf1C=3I<o*g{r)@6RNA&QKzJ*w;e%ntyJM90?eXc&C9|`xx
zVxi9rczl;lwITLrgM64Bm-(@NPZjjL_UN`uGXin#-)$J}A91;<t)W<UgS!8KR`0KI
ze+F#a=nK!PGMQ`h$c;rbc72!Fz*UorE#mlvNu^x}cb?2??5(<EXrrZiXDFw5AHzWx
z*r31kvDkgRlLm~(Lwd~V`qYgw%?+zc!5$?LY3CaE?N;?Q&-<J>o^+;RIw8&7UhOHp
zBiwiVc&JY8o91s$RObWtZGMAs0^Wo^uer-Gr&C&>rI<6I?CMUi0a!Dnte^#lJI4a}
zW3aho4A1}X@7@MtfdB}A00@8p2!H?xfB*=900@A<JDGrI{{P=mIX?SNE(>%A0T2KI
z5C8!X009sH0T2KI5C8!X2qwVR|3A%@xZd=!KRveI^W~oBkG|6VJKa0ok!W}1$dUi=
z$afw&(eb_QKNk8|q2adgYztA;Z{ejsQM(-B?&wcNdq(M5D|+;t7o|1pF%DiU^PVSi
zc)7&;pXf=<rs)ZJ^N~b<2)~q^PtcR$(+jhixp}Y5^nzFBg^~Ek=umuWC_c%@C(g%*
z&yP>)UA1JGyMNzMsaW2UC}Vn9%p+uw_dPztmm3wESz5SG&kq?369OJGbBpMIJ}Vcb
z&2zPxD5v-8J-g|{B+?@!hp3n(aUTluwdn{a>la8rK@|CXxlk>Z{DiXffDx-re<cY8
zdcK}F9=@@cLby+3C1z&q+G}|7X^)4M_#lYaYnKS(GrDXkY1U|yL0QUBeY#cD$kZ&K
zy63ZVu7n1Lo>eDqaZlyk_{~qr1)6E8mY^~X*O*~2@DEfbt9p5XD#mZj2PmH<>;UDo
zQtq$B^-%3&5pMeR=c&NvBTfwlWmcc>p{>+qV|7S!CAFMhZniRde&54A=#f7oy!q)Y
z{{z48fG7U^Pw9C}J?Ip%=^4EsyDQXQ2y@GuR0pC`u2?ld5ap7wyvAPpW7dV?2u%k~
zHQ}#?<}^Y6Z?pE%2&X@LoEf>{FH9fOoY-o+Y>kKQgUI6)B~h*xHECO8-A8eUR)#Dm
zXOq*k;wh+bwT{|FYE&0$XM-D6%WR!4z!QV4#%N#GQ{$Faw&!SmjImY8tQ91F$u?*2
z1Gt_T&Xek=a!q2HHG3Aa1#)PseS|nn)lMG*hc^v_yG-x^6!wNt->F@Qa4&u4Rbrq>
zK9AE(dcVG>IpqPLCT*%#?;;F)ffjp+v?=0V-t2K(#1{){0j1jcFn7~%b5cRlq&MBs
zeY`dm=B8gFTMP8758_XQ{HT7cHW}e=>M_b7;SSTxm3{8e+BMIB(U}{V<Gtgvy_W|D
z4$3BK6Xau}`W4H^R7>)msy*6v#%6A&Y@BVb8#3G~bFcov%067GwUEtd0`9^nH;?A$
z#Dk1OzgRm*s`uy%2UFcx;dEf-t@TlRs&B243o5{;YU4D;k!)jY7&^K*S|bmh;<zcj
zkKzZ7$lGgU)UUU(0l8uC`&Ge*Zof*HBm&gP*G5T;?$)D=Lm5@*r)nb+?p1xtvX=dp
zwb{a(=9Iy}JyQ%f!v0K~&HvvW+U9z<kNxeQU+cNh-PM(kelhY#k?)9fhyO<SeMi2h
z^Orgk9lzZEXYIE_|BNEy1p<daV77KU%w<_aT$8tXXU(}il&1yoRu9bX2$`*Nw}Cu$
zcef(k%lb*X$=Qq8ZGq0+2v=S}kH-T2eVwCO^U@l3P8itTy{ooDqe*?p(#W{8<*dOE
zRBK-f`Q_}Q)o`Fz%e9--llAF$Y*5C%E~oc3=s5lEAn#krX>`N(0?vl+tjSXudRpDD
zPl~eu-aCkAv3(B9mxQJ-cDA{|(c5<IMuc0?JFP<5kq=JUmf+z_%cX*de!7;4aJTfR
zm9<argXixPV=o?Tp@F_wOH-XbsjK^|Q)@Z0YV7!7)TytNZMp2uoLG44)wi(O+#g`q
zO7+F3O5J6|^6A<#_0*j&Q;`(OxGU=$F0#Euybi)+vcAH2M+^;H##px5a;#9PErq#j
zY%=J%&1v5Gaqn$18vAOt{#!;ZYVy_EVuZ^$W^CPB^=$QVfQ&t<_pqZ+)~-jmq~7N*
zG4sSe=ee)q?y-SMP(%87Z6VBkx^Za603XhvEK{4OR`Qg-^FXb{+!tq%H$C@0Mrf_u
zhY?~e^LGdH?5fZ6H8v7YvOwRbKUbTJaAm#4Ei9J2^|1Z{8hgVWs!4Aa9<1QW+H8bV
z^@;s}|Caob*_J({;U*<_@i?yy)viUj3*X*OYWi;e7zB-<E%v_EY-M<mJ)KN{<mqrc
zE6J^b`SzGF)`&m&E=SlR;{aPS1>5x#wX61znMN03?H^&TnYa6XMJli@Nth4y{qPdt
zM%|x>ymwpdR833qgPf?IsHGy@j$ZV&HD;C81ikU+E$2qIow;S>P2)K`{E#=g2)bLL
z=V~vq+by-HTbIHp61kkiEsrgAe?M(`u+m`0PD>x{Dl_*83pSpP+LZ_=>0v6O=a+ES
z!H}icvfLYWHvR|t4V+1{e*mY?L99K`{~yAv4kADR1V8`;KmY_l00ck)1V8`;KmY_D
zMglnh?_sPJ=mY{F00JNY0w4eaAOHd&00JNY0*6ZgzyBXDPe=m+5C8!X009sH0T2KI
z5C8!X0D*^*0OtP>W350Z5C8!X009sH0T2KI5C8!X009s<TmqQ?A1+Tw0|5{K0T2KI
z5C8!X009sH0T2Lzhmip0{|{rWKqn9Y0T2KI5C8!X009sH0T2KI5I9@{*#GZvc|sZp
zfB*=900@8p2!H?xfB*=900=yc1laulskUQW&!>*Q-d*VK=+dJ9DVmG?T%<4ji${Ls
zNKfZyJO1B}RQqQ`a@)^Qv`74<pQ;tvk)G4m!IRbr%&Jrpv&Qk)YL-rCq|?$%>*i_L
z`EqGZUau;GCYMXTBRvCS_|KO2i)}vnb=L#tr}Wdc0y|dsitmujR=RnP05%KN^5ow@
z#jVpr&C|UDV|dOm4m$91s`mccW`tX;Un6zR<8`xva<Y4#v>TX=vOnxYca0h@3MFA(
zQdG~`*~Y=UbQ&w2>Y5d_=4xZ%)+xKe;emYgkJMg?a8-TXF+GoEWj6$M!#+{gIgPuu
z@dX&C-}HP*4e53Pty6~`+Z!mAwV@qJ8_ElXeAPJ1b|B#7*(v>lH95k)st-4sV}LH!
zf!@xc?N-TJ9X;z;);=fMJwn?TzLy>9@2+h`xR;&dvaKd=N_Ed$^~v-#?11w4+1d)G
z=F`b5$t8Zi5otV-A6Hwa6YrDe0qByVS;yP^6fv(^HktJYSp`{B%0=EdD4OhKE;*CR
z%o*S0HNLo`-YG06XOq)uJ|1uw?3DhV+FFFm8g-ZuMcd#+X-%jWv_pYo$!cyjj==@e
zsXbkjBHUa(;d+mNdf3}{1SC43=P)D%)<i=~vd-ytyNT=;`4j3cg5vGwRs!8dx+aFX
zGBcVrd7GEjY(=Q5blkLU4Q0i$BjZS6D`&I)?P>|fYk794{HWcU8wEQ~$SYSh1qU=2
zZ!CXnaa4X#lhFHXs}XKfzi6pwWU;Q4tCg%QZrcC?^?V=&{c?BFayYPAx7CCQ_mbXi
zX=&9<L0FXvgsU7-I}fCtDabaZ%<OES65X{NJ8arMPuo&LD3NbBMlOe;iAUOA>Kv_Q
zBV0xw@H1H>Tm9C7kd$5ODA2h`wNH@e?KM5GCbe!cyeMdb$Fho&C|8S`v~BptJ>4rg
zv6tyw^J_Kc#n_4Qa!Hh^(Ut;?!0jRp`!oBLLGxhnl2DY?ijbGG?0|2fB$}5E+qbk+
zTlJ;hhUh!W-3ahWHiHeR=D5#vo?Z#8v-Rj_pMvfaV_!4{1v)XkTKhPG8LmD1t%0#|
z*ly0WbDKqGKDC%h^47`rnZ=|zq&JIcC22-NGF^x5jyd*0fl^Z0ZD$`?@Yg<7dnv+Q
z`z&ji=AgKxDOeSnPF$L$GF&!B70H#<a(bE11y;;KhQr>vBvAhG3+u>?2&`b{5Tki$
z<3#e?fIqqu$ucfufMUyHffYfKN}9_(P!&nl$_kCa*@&mP$uluF`@o`@ys0EJiF7iZ
znoBwZHI`J9wSv@8N_L&bT%yb-djVn0I|N*oOXO3wghE!8SZbO|1Qn-L&a(7bl{|7|
z#?IJiV7@#<kp`G1(zcY(N{X^aibR3Nl5%N%pex7%C-MCMBYsN*#RCBl009sH0T2KI
z5C8!X009sH0V06;Kg<9GKmY_l00ck)1V8`;KmY_l00bU=0_^$!wqqaT=pSAn00JNY
z0w4eaAOHd&00JNY0wB<mK>cD{l>1EV$&P4r?$xenf55W1{_{MAMEhxJmL^$W=$n+6
zWO_iKF7&S}!Wyl!9kLb&4oO8pF7yvlq9}#26<w=46z?TFVs@=u5Gm;(Z|29=8uY(R
z(eJY~j%byoVOdA}jTn38WN48!7xjr*TSIG|r7JDkPb(mp-lmGKbVG06PG^T|I~7*!
zeov{Q{jxe^uBxOEO{q#oVWa)3q_KzfSzd<Xq&h7WQY2NC*tPzTNZUfOQjk7o2s@K6
z7eCt13K+f5-tQX>=<R;j{jRNH@oT^Q6)8SGGAT@qPfU)*M@I!|Vs&^tKNg=H9~LL4
z#wN#lx>T+wG|Y92(dv<RMLyD5?$~Mn;n0tUhL4>N|G?2i*LQP&!QJK-TN*%PK==pR
z{=IRnOKtn@a9`?lNKY8+oY;b!g1o8-%1%})NJX~5!(7(HmN7Mia6f9TcWGX<(-6}m
z!(64*5bjwIJeXrES7M~tN5g%IM5un>SZm&pfK_)%%Cj{dtwXrK>sybLYaR1|%S&xS
zY^hDqa*<798XV%PPwOCCL@}3{wpjXNEsUWy7mzxp>F*2oO`Htr&p5SeE<cisv{0V5
zN7$5TBLrG=k~J=HKiIM&J+aJW-c@EsVpeyCz+6n<(Vq$Toje(;PZ*WvN$mN^{eZt(
zJV6Jn2Q&NT<x3So+Xz_eQ&kE9S8bwfw}8OaM$&dgrbcdT7tkCbreD|J8}2Kg4%MD?
zpjVXg7LhK|DnRC7n71k`qVMyjv}3i7hREFi(y~$-V(u>1tgwb8tkOKnu;qWIYNQT*
zu~x^&bVctA_nkf+sy|~?XoHl7uiW?hE4Cr{fR)>7r9N9;<x#r1_Pyn64t14lw1t>{
zTz^lvZ*C%_C!LxM+|R<QxWF*(`&w3CV0=rKSzCcISe@AwQ}5QF3ipkUhF-tmX=&tD
z+!gF*v+`Zy{wIG01w=VuB^i?5(uMX^`}^3YfMT_>A7{N@oFe%!J@8f=F}+WJKXs#{
zA^oCbF#&L`It~cozPF{d1jP2)N<g^RQflM+)6^GCgkB39mJ(=Mffu>I<F}T;NC&i+
zR+YjUwNW6;mJW4G7m~gI-F#yDJ^fvgKD}o+RC~dw)YQz%z7`|<7S(#zXPXt}U6agx
zk44y8?)<Ih;O(2e`7nF%t{Nt9UoA*kpP~&bz-e^&K`MP`B2;rtx?upNr}Phm`(*0z
zqK?%!M6oQ@nhVfa$H+9x21X4rxgT!X0vclOBGUxW5Z}`p26N`5gblg|bFBRI#~7u>
z^jrFq;XajWyK&6qCAsU;^MDHe?v{mWj=Zbv2dr8)hW0FLudnIP8U6UTwHZ#Jfl$L&
z?uYyhzajWdX!35Y1dgLwX_|X*qxzaJw;pt}Ay|MMQBw1YTxqdhyjT{cU51M-#)|e}
zGH`fKe#qK-szqvhW2#oETQ8Z9)fe?=!+q1!q0bDo8W?0ZWVR@O$F#tPp3e;8{#U;R
z>=N~mt-<Y;8@mGkPD%mw*iaJMjH@A%BRgojES8;Lf<w7!=L5TTTWu`1kAVrxt{|vd
zR;W}8vJ@}}bhM4f*z^Bwz2D&IA6_5;0w4eaAOHd&00JNY0w4eaAOHf7Ac6MwXcXuF
zJ%SaBB7p!1fB*=900@8p2!H?xfB*=9Km!5n|JMKnUqJu_KmY_l00ck)1V8`;KmY_l
z;1MK%-~S)MN=1=C00ck)1V8`;KmY_l00ck)1VEsH0Gt26o2zlh{_C-0NB`nz_UMyc
zx1#?pnvJ{>SqOh`_)6!;I!f&y2%T*ERNHCp_bB3_zi!ss!rW#o+B2FjluHs{q<zhJ
zk+xV^Eends8~a13yjJF`C3%guGcd2vCW(34K7%jHw0R`kUt=UbGCCBW8j4Ty@rm>C
z;q&8Dvvn@aWsj3&p`b|$->}&T<wW~$Fv&$pl&eKe+SZnnv&rc+AHNVF?t1OhVeX~V
zBu*}|Z5mX{UD)P5dwlQ(skE+bFp(qivB{w!etsd%FC`Z<siovhfJpsB?NbphuTL`_
zMUfTpPF3QKEpM9CH8(b5n<ci~`K6(R_I5VLCMOH?v@L{x|3%;SH7rsrsNDM9+Wjb}
zKUsUh(lSsz&$fxYyLsUd+cIT{HovfTRHWT;_N}}pl`GKL-^Y`evBzOR(L7ODDO~~f
zt~`TmOAc47;G)-NYoCm8!fVe`$<3WP8f(Rq4{z*>!fy@xHaJOLHjE&-l3GqLH`RbA
z`2lx3@suoZgPYcS*7(aGnAqBlq{Ut`_Fg4{d&vZw(tPdJ2v@Cj?`BE7Y3M`F&OJkH
zJ08Q5_H8>>TX96VdqaH-`#1$Pj@R02_afZ2Z)092*g_6s76-J2-L^h+Py}r1$#xCG
zK1fD8ozKk9vJG^c9eSEJUkx<u#o7*aHLMdmyzWbn*R~_vj(*K*EZ#kR_<XrgEtXhM
zv&#lbX18a$C3EjP=5O469PRCza(rTDhM!)T&CIb*VxM~lQ6JaT<?0`--HmWX-R>%U
z9a_1>ns#cYh1M+_cTfp!Q)jL}Al%cl1=g2T+lp{{C#x{(^%WtnnH;UX0ST`M6L0Nm
z#q?ICia-Mn$!yho4ngkMs$uSap0W{@a>ef7O$uf=N>Bpy=jAnC+NK?|)S0D)>*Skf
zk}G_UZJ`_x+1|4;pg!)_v@mza$-h)C4OvyPw+`N%-R%Q?WOm89K2}pB+{>?*sqCg>
z3{>4hn>CDdw~jNY+*aa4_LlDMSdaKVVSz2<T1^RaFSG7UtyU^!MdN8yzD}d(%+jn%
zBU8&-js2O;MzGtMS%0E-C&I}(8$dM;aLM$I?Uh}B;E}qAH!x!Ey=`<*mtE_tRU+KQ
zdd9KL<}&SViS`E7T{+fs+P%c}rPN$v=@x%2d5ceE(hI41k~Ei`Py0HThhkIdAT;_X
zYGoSgch{b>>dW*|<o67(A+PQ5r`(N^2iaEl_NLQ=*03#Rbxmr^3Us$e_U%^f-NoA7
zx!N4AH6&r)-IFy9>0PDhi`6#-ns@DJtrX#s^$EA~8-$xiy_*3vBsgUE!<>$%IL*6I
zdQ09M>MGZ;MPXMU4?O>W=r=SV5ClK~1V8`;KmY_l00ck)1V8`;9y$Ve{{Nw?5-0=$
zAOHd&00JNY0w4eaAOHd&00M_j0KfkqI#LJ(0T2KI5C8!X009sH0T2KI5CDOPjsW}p
zzwOusj{e~V0w4eaAOHd&00JNY0w4eaAOHeE1nQUCy13|PVslS+L_1#XiuM~@bXYs6
zSR2jspXVtg+OL$Wnq+;UZ&F^8X@?!U&`;a9tqCPzNR-xuYC#*4ih^9|AEZQ43M*GD
z+0`A2_mUkkYi?xJKggT;vAv=CU#95ySsF*Q%F?i`qy0vVJ##X&$eP-VSzAM^7_PKv
zKkbyn^fpy=r5k#)4H3+e83l;;vrP?HY)`48{jxe^7M4OZr7Br*RY}umTRWDQp*X2d
zTV^Pds!Hry|3{>4p;#$MA2WoV$(M^C?Pmpy-e>Rk4F>dfzw3UN6dxa%6eh+eCdcBV
zqk=TCIy{~qi%*UZi<47hlVe-M;@5upE8^F_^nN`5A7t9N1_B@e0w4eaAOHd&00JNY
z0w4eaj}`$u|Nm&!CrSeXAOHd&00JNY0w4eaAOHd&00Kb-@cVxd6<h-W5C8!X009sH
z0T2KI5C8!X0D(t~0Dk{}wCWS30Ra#I0T2KI5C8!X009sH0T2LzAOe{G2T{Q_5C8!X
z009sH0T2KI5C8!X009tqv<Mjc|My<y=pSAn00JNY0w4eaAOHd&00JNY0w4eadlJw`
z+oGB1)6Z+8qsg^3Nf;4FrJ+@6a%yO7GQT#oIx4IUjpwJv&rME^OQR!WO1Z45QeKg?
zYtqh{t>F*fx_RT??9KV&t@RJjuisL%331`)zUN>5-pq|Z_`t9J@bkYi`9qg~_T&$J
zex&`m>UT{(b^S!@mnObE5+4~Iicby2C;9lq`S{5B@yT5;on@JR%$qCw{lBgEcR2co
z7YKj=2!H?xfB*=900@8p2!H?xfWSM9z(9K@x}UocJw0FLp5or%LaC#FdGw|3!LB!=
ze;WCBkxT907XJ0{>CT_-6gvLA_x;DdaICBC-*9iVo#cM%App_y)zEMBO&kyD-Ev8k
zwo@}J*{UijS*2W%%ullDT;P6TdMTMmC;8O;Omc<K8L{=8J`(Obemqp~(S+54<ivJ9
za6fM)OD8VPCi$EbkmFBv<zz8O^3usG$tC{!Qfe--bc?^1yu~Ln>4nrhrJPI7r}_B>
z`ad%}+cn7N%F24yo5~wz$Zi#A6slDq-98dDiIkb45~fpgNjJ~fral~QprlDc(WGR2
zl11kN_jApZoWQIVJI61~^Eu}-kublIcCp%d{dl-9kqFiA8$=88sv;;mS*aiurIN-D
z(kw}NO)i&Ohj4$_&%AZa16G2qOe>t5WKV9zdaKA)zhy1%QWsSJRw$ij8jwM(wyqC^
z`%at)z20Zgt4K;wR#l?qe&T-8Psa^8AdOuuOEBsx$ETAkenVFC<%$$=efQ#5D`G1{
zB{(K`Usn7~@^T_Go8|`wD1xAAio8l^$Tno`k0py_Y4VyZDXpW$3i@EUZ~W<y{*2)~
zGK)#GMVJjV%T9}BV7Q;MXlLeAiy6mz1VuNS1xv(Qgw@aob4*e$wyd8%(s%mlkm|T9
z1G^c^$mkz7w>(djK;f}kMgMTP@9C#QuO|$9VF?Uj|C`@N*tG*%hoOFR^^204SL6z7
z$Tnol!_k4h%wPk|doTCLnxn7snZx{6>`9c>N<r8$D$^ECT9Q=rNMUc;%<CJnNXFBw
zyJKMSn0`W!h5MFIg!EgE<+yFoG$L8dnhhswG!r+N`v)z}$&C~!!!!Ul60<SGK8V_2
z&T6KEIkQ>D%KE8r-xTTC>)4x{)RymF;l9vpd~T>6e>W9k`ZM~6!hH+Rh4e)S%jQ;A
z)yj&L^)xEABIKp)rnF-Ra-VOB#Eue}-qgm9($wUPe25I?M7>hC{(QLax#vRlb)%iw
zsqDYpPx#x5ef@xK#%lY{(7@Y+Fq<=VYqoW!YxC7Dp-^pfc;8aXh}Amv4~F}meKu6j
z8aT{fg6M#0YHXUy{e3?S>+)O7f*qN?7Yxhe9N0y~M)Z?M`jW>(oME!+or0`MS)r<x
zjqh3Ns|04MYIfN9$o+WjwBBzF8ecnU&~t*E58RLW2|59cp?MIwS<T)tqt{4SbKauj
z+ha>_SgVe=qLjC&=WFcqn59Uiys}fF0isc&9Diwc;gUN-{MMXoe?eH43jKbwb=>8^
zKumvHKN0Rroet><$1yaH=q&$0XR0@Za6j78IW)wyeFVucJwrpd&q1(AG5x;INBZ>6
zM5uPLrDmQ1pF3*IvVn?cC`)6(hTJ@X+>h+L2>Z<}uz;Q{Sy^3enG+;^>5oS~13*)_
zr<=y=`bj+=?#s}u>SYItGt{>sTN44hV{pQ`pKV!ICr%3)W|^E=zUs2FqY~IXUTi}@
zOT#zPGHhtcW^+zjD+lL0_cNAOo~fS`Xhm+FeT*iFXS|a{C#my+`)Ml!Ym(>$u=ak)
zlSD%{v+*F`VB-l4+{I1C*z^B=+;?)vezWICd&ZCcqoYgRzuPTzor%64?Fj#2SUmEx
zop(ErcKmS13++GC{*lmR+ZWrS6#Y$mZPni!;imPTXwRr1io8+Byo2?O1!1>pbvU)2
zt6pe$NMiW>_>?|g?~8C{J!|D$Ey;JPQlkPbZ1}A*{vgHG`g<Z=LXWl}W_sA&fZ2nV
zCSF!REGX9+SAROfP1R3Rwx)d<wZt<+<c;3Z*D_L<jczBol3GqL`)?t<GulZIgPr=j
zBisvmpO3uJ65VdCrDZh4Gu;ro=yQNZe-)Is-d=wy!Y$~}2AQi@HXC;CO9^vM#EhUx
z3nM7d4D$7Ng}I4p6M-|W%+8})V37lJ5$jJ<E_bb5^5sIcSmNE8oKVmt#WXrw`NYf&
zKfN%UnWN&k5$$0m4IlXc7qC%(BFtTQ)t`-bD!E5S-l%(IRr_eYH^QBLZJ08%nhzth
zcgwNr<{*v6tl*}3_QOA+53~t=pnfdE-PB*Qs>|oE4J&Jyi??h9&GTK~9NsijHxg@~
zmV6qQG|s;R+bUP@iEzvMlR>Jwg3nk|Ek!+J7B`MP@Y+qOKwP!=){jQG3-w#B!MRfR
zHaj=&0Y^4AvA_X}1<^m1VDpz&Q=VN$G?Z$WdyRo?K2`6IaC-a8gvLzDi-NkbS{4*B
zOAeS9N+MqoN>YJ_`}~?xF7nd0tZH&;oq|ebO<*^}XcDdMmaZtTD@Mcs$D==A?}~6Q
z>C2XZn4=#1UWDaKhgkbHZXgWN%(SI7+o|l4ij#`XXHB!QqxgCnCmx$Muv;fVmRvhq
zk4CtQuXVasWDB$W@7A!TnK>eE==bf&2Q(skq!Kg&cJ*&IBctR^j_>equLL2h)+1r=
zCDubK64Q}v#Jab@lQmg=*{YD=<h3%-A|z(h^rMnBp>QJS_@(50f_~qzdo?x_NH6f_
zUFHCr+Ny`c+)dV*HRWrlVdYGj8uR#l=E{<q`5LKm0T~PRBVjI^Vir=a>;x9qAg|Kc
z+A4D9I{W#`6LmS6Hdyc%2~t2FpR9LAxRm}BWx{3&-U1my9c;cH+qs(Kz2gI`Oy1u)
zJhw4|1`3sWN0?g-tW&QlR61FFV6arN0VS>t)Y~K6c>O-(YW(QOTes)-cHWQheY~I+
zSBup{Y--{`<<$J%u$-udkMh{YJN}kkphq{?|Icu}GjH0I9)dvt1V8`;KmY_l00ck)
z1V8`;K;V%epij|n;?WZ)^qtYs>vVfq(l+RUa&{v@EmxJilp3yRH;dc1ZzL;9I(_wq
zc)6h6NH1N_+)La|$Hj%!%=XIiO6B&A;X*uJ5a;JJ<C&Wje&h0OHN8@~IyXIc=jILZ
zrnI@ek}eM4zP(dfnJ<dxZl>3_SLbicsLNX`>+`3F&t1Jbd-vArsF==Yt}Qm)L<-m&
zVD$XhB%A+tc71^B{gLjYU4O4@l|JGH0w4eaAOHd&00JQJ*b=xzKeI-kf39bARG{@m
zT2Uwo>ui-zfgUWERR0Kb_#8b&8F1-J2a_?-AcG#8poLm_^U;ZbQ~{T=^aE(*?DONJ
zqm@E+T`rxeR96dfo_^d{3xcX;t;asxlxcHMtoZP4_nNg&R(v?%(r4cric${afAi^p
zEGV0y3297>=g$cf@yW?^=cI`t_PUh3lA7nQU%Ad-&s>^KO&jY#=|TJHYesNS*9~)B
z)OmiPB9$oKnO9VPYwXN${LFAq*L1namPelFH#DuHo<Dn*7T1+3>y=kj_Si>HS5Mb=
zWiGi=Nw2JI&t1MYmP{t5=aQ+ri<jhu%;Lt@mDQQqt&*6GuO(N6rTA2Q^6tvL>sKyU
z*5cRWw|lx?yc$2JsdrAtH{|uytrsg3H`DW1ZzOMP`Mc#?v-#^wR|H{dVe!stCUb7|
zQgY|ExSrU&p1wD_bXn}_8b6)hoVj^hzBiK>Zd~5Dw>l$^$KyNcyVAB&NhxBgba(pf
zE6Tb$aeL?N@c72$wZ!<;%1UNaET((9Ze2Q;UOK&ym|D6vmpm;_UoGF8zmgGmZr!}S
zv06;7U%zsrdOLoplrLT`-nu!yF<YG9T1`((ZzUENd%BW$%ZXGXGi{Xp?ycGB#e6Dp
zH<i4+xPC3MzIZdSo=hY*HZLVM)S1M_*ri0Gd^P#v`25V3%H5tW`O0cZxLKZ_Uc9n6
zy1ZDL-dwt=TsgO*%x$hLTw9w-uZ)!MPOVkTNo9WQ+$HH+advKOZ8*6(bDQ!`SMH21
zCnrzO?Mz-8zm$4$ba^39aqh{Z%i|MQU%YZjyqq|FH+3~}>*kztbw0Iqck+s~Qj$)W
zGv%JH`Fr;g)8f|Jot4DxWaaFOm**BYrmw9f%QvrFO-_m<w~BXfjhr66QeC^TlunHo
z7E8AmUYt!%&Yr!I@97#|9-muDF28cGlDFzT<*Rq?#g((mGvkZJ(XFu?QzLgOLhAP7
zx|UvvtHeBi`T8rjZYEy5qb?51$$910{MoJb;k%j9#7yB<BE7XEE-c>K+Dz}H&uXhP
zTiYX-RcZ42?Bv+oVuEVy+O@@ViL0aQvp2SiQn`9t%*01;m2ZsA-4fE1*H%kcGn=;;
zZ*Sfa?~N$(LSbilJ~cIcWwd;&xYg5DF20gUX!q6=>!s;@Rav_-dwX+pZo5#tH93=5
zEML8QXEAx@(%iK~V)JsMa4s`EJUM;k?%l-n`X(`7+l<c%rRmh|YCir-g4o`;JF+d#
zDr2`7?_AGpR9BVE&gJ}_!p-u{NpZ2fD6HRIm{cdLtHsrwo~~=v*~HZ4n^!m2s@0`M
z<@CbT`iq-0H>Ra~qk^m!C+-!mUb>T7kVhwl`NaAwS4UUN=L*vD^2jSoXQ_>i&)<=+
z-H<0oHF;&`dMQ7%c<cI3aiKJ~aWh_+n7)yixckb)_>0Qrt*ydR>g?EDLMg1Ow@0)~
zC93b?{P@<?*4eR*b9bb~#fv>%7iZ>ImwLJk2auef+08oy_@RJHH`~bfotQW=I_ln5
zFdyrfv!5$lrsvSA^uS!PLQa*Iv^PtM2V7$N|8@3W=X(E5@8^3z*Skp{@d5!5009sH
z0T2KI5C8!X009sH0T9@O!23Gsera3K-O|qE6g1!q^4$dOIN3>eH`{i-x7mK0;*2-P
z3A#BM`d}yBF^KMQzo6&XAlGqph3om5p6R1sI=W*2w?{D%3<4kk0w4eaAOHd&00JNY
z0uBK+(#5r%R!2vBdU`_6b{gt=TDmTJzS8y@=hx!&TjluV>d@-Q)X31-_(XnaQWV#Q
zro>foZ6bdz9v>a||8hAzem;IK;F1>Z;-Y8GOg)*1v?U(xfoSjPp`Jh7T-76#>rhj!
zcNN-G@=eQB>7-08zMf3pRJ?Rj+;8o-PR~hs;oQi?kRVQ;8yXv(h!3sCC*wm?tLLUB
zhgXNst>#Dl6-bBIjgAFeqD-^<uG2%zmHVUp*6Cqps_nB*4=vBG-v7*X^!_&2`<325
z=>7fP-?smA8ZthB00@8p2!H?xfB*=900@8p2!H?xJgfvd+uPbgk&gCgTbR8aVQ-zJ
z?7Kr9^nR?J-l8FT>*zZ6yIe=l7r36UAN$T@mwUc&?03!oC=4$U009sH0T2KI5C8!X
z009sHfwz-Ds*`?#Z##rXDk4`q>38)u&%*-;eoi8s>ZBjby`m2K@WYW8JLwm2ucQM$
z(Lj^kwt>Al?c;ZJHevpKpGv`;_92Q*yZ@F?R3N8)NMdvM7gc(=-#Vuz_`#Ze2x9yH
zg?hWV-q-03FAx9$5C8!X009sH0T2KI5C8!Xc;pG}gpP8{J&|aJtyzhfTb+vMSF3V?
zPI^_(Z^_bKNjdKxgsak4nEBz+xHu%Po)d@0^1|@Y)bNx@>sCbChBJRonp|z(Wi)V&
z!%o|-Sxcg<m3>*S4zG@^jZBOUjZef!hsMUl$ssyIG(WUDF)=Qjo5-)NjZX%wr3kph
zCjFt_H@Mz!&>LPL00JNY0w4eaAOHd&00JNY0w4eaj}ig49w5}-9*vsw|Ic&1ztj8q
zN2!1)3kZM!2!H?xfB*=900@8p2!H?xyn_jx>WD74?c>>m&;Uh@HbrcHk^uAncd#`;
zbr1jn5C8!X009sH0T2KI5CDNkmB6X?Oxu3e9fStjGtvF55@759xhEb~TqqX^fB*=9
z00@8p2!H?xfB*=900@8pBVcU$hxtE@0R%t*1V8`;KmY_l00ck)1V8`;9%%xY|3A{z
zi^72b2!H?xfB*=900@8p2!H?xfPg^&^M4ou2!H?xfB*=900@8p2!H?xfB*<Q`UJ54
z|Ix2wln?|!00ck)1V8`;KmY_l00ck)1laul{oLocW8XOTnVv88q>s*b|BLSBu3ze!
ziGD76I`WH=XTz&U{=<=t&Ohk9-SJ!PO6dDS=i0v7_W8DQ+XuPdri6$0O4Y~0+(Inc
zGb$7$MUzYGe5tI-YjR%D<Z_9xNJ>#wRr;vr%c8^=<#ojf8HtaK4#lU2;*)%Q;(UDg
z{P@(>`e>M2WU@r1T;Z*Q5csunL6j6euSkL>@j{---CNj;^^q_)X9yG2jn%TCh+bv*
zsx0y(r6TPu>SBF3%uTcuwP(4e>+vu*&C+X<P*i<tI6U^0^3nR)FgL}dD62xA7iCpg
zEl8q8w5;qYK&+pMaC*;iqpma}sxbllovNhlWH)40D=Ry^P+%eXa-mu*S;)og>JG2%
zR3u(2^Q-c@T+)`4v&rc+AHNV#i~3@HD8k*ZMOi+g$jc>B+V<yirmU=IWpT)?QYL>$
z6f{9CSCzb!rJ9*uN+!}tJ~cm+T;X%6nU$<h%dD9`>!#28)93hwc|PY$ljBe2%w)O2
zoSTroG$f4$6{_}$`e1~+^4cvblx2uM=@!eaVN3IL;?gYXf0>_ONb|{+)N*>cS)Z<)
zEas??>ExB<5`TRuHJ4br#a~O_;uD$lLTa95%_Zm4hK%{l>}=N{pCdRqKAl|gU$qM6
z$YQzLDH@8`oY(^+P=FmSXs)u}4D#h-Q7TbjlUfXsEUAqlfrRx?{d9!8raxzyjq5mg
zStRk~Qphk3V`{bzH!_)(9oaMjH;7q)Cp1`#j{1k07AHsxH>Fntr@@*n#cV@4H*Stk
z%*^o93$vLyGC@y%emwk4@^T_Go8|`w_@G7<s>dQ+N@tDLmCXy9rpT*Ra)S@1hLf(L
zD)!J~yM8Ldy`VqsS7MVY;Q{k!NggcgPu2%mJv?bygXNGmWRa?u`5~oDuFHV$s|R-3
zUjt^6X3KSo9uMR{UjI;pdr4QV{HrCIjM?&Kt5vx`JzbV%X1<k#qGVZRW<Ir;aa>ns
zF_~2*Nz4k=2d<Y`JIMwoH?6fHE;Ctgbn;e4rYyf)umPW_KOf<ex<HCsj*7|T-TX~z
zyZNbRCA$1t_)VF)C0DD4us|g@>K_bqFWe1c%LJWqdw^z|MoxwoWG<e4W02T|z^q=b
zpA2)$bClJRR4i{vysst=VOD2CB4(BruG^gstGR~YTu?(DulGl|vYrhpjJN9SKFg3P
zi-zo$CbL&U(^$N*eBGOshh;6nU_MztK^>aWw%uOZE79tto1LvUl2xi*^$DyO{b-$M
z4U9S31dS4`25#1cG0^ZdvVex}Rl?ukTlIyzH~6jk@d!7q_gK|v4qCjKobj}A&$z@J
z#i-L@6s$&q>*3S&4=_jnOzVuT+T5ce>^e6_K3=VuW!Qm%RZ$DopCdP4XG2TNWmxHX
zmTqr<(U99ghBYjsh7Q7UE8anpZ`@%_2&i9~p|N5*HJ9`clqxjvkpe>lDQwl>-)sO?
zRZ@&vJ!5<d8kjm!vQ`w+0G!K)cVc2%c_@9n{%lJR;)t=`hn0!xnjC$a-Q->g1GKan
z&S1{U!|rTaDX!Mv7vU1Rf1K~++tPhFF?_zmx$1Fbf%R15>d!>DsrqTxT3OnCnyNRO
zgV<>+=VA=aty&HoMB6PeFwh!d2U&6;13dqKcsDR02?Rg@1V8`;KmY_l00ck)1V8`;
z9##UF|39oX1HC{11V8`;KmY_l00ck)1V8`;K;ZBQVE%u2EFlR5KmY_l00ck)1V8`;
zKmY_l00bUZ0&M+%sQ0_L-hWPSc!2;2fB*=900@8p2!H?xfB*=900_Kw1ja&P?wPjH
z(dZ3Xx+^K6hRtF|&X0}-TpDeoR8dNm6lL0gEg(j~CHDJ&sQ0h9-fz$wULXJhAOHd&
z00JNY0w4eaAOHd&00NH?fzI}}wos_OJsLIV|9`^u{#EavJVFITQ9u9$KmY_l00ck)
z1V8`;KmY_l;Bh6;)&5MI^*}+i{Tb^i0dxNU2K)X0jmH%X$_)Y_00JNY0w4eaAOHd&
z00JNY0*@$xj`nDq@%w+zks9}I?n_+H-#Pl#qu<u`;pk6BUx@rl<OAVfJreHtr|o~$
z-V=VJ`?E)WyZ3+W?K}1&_obt$?j3{46(5<<a@)ne#L19;+&J!(9kZS<D~ePQG`U>L
zy2r75u5$mf`5aPDuqTmu!lWmRo$Tn!%*EFAWVmni<ZB<%UkvvR4TZivW}N3N3hKsc
zSy04mrMg;>^NruR-?TI^&sgto4DaVpb$O0S_a3Y6Jw4P-W*jEmbawe}Cp~I%QKDm=
zH97=(-8tRdI-p&XiWLGVWw!)H7FG-H@#j_qA)l92HLI03rBc9=r}k;<t2@nSRvV{G
zn&+Bkec_g9J1sk@eYZnVv;GKHYFW)nC8jYQnra_PZxeG5&t*CYrBYcljAUO)D1}9=
zAmt6y+DAT=NZk;W##3P(MFvdk7>MbTo(T614~J^1VL$B9`JL>BtkQCtoh)5bny+yG
z+-F0%<``tG2RR?q6clVx#@VB-4zq1#ZccfJWBOJ7W8uC#1EE^jX|I08O(9v^3|4ot
zbOx^AwmDyH?pK>zvM+*t<ZMfEW()VF@toc|m~#`-QL=tba<SSA`U~N{fq~F#7n-Zh
zCzJcXcB!lb)>Ci|wW<VW^9}kp9CJ<)Jf&@FK1_fe&e~pM^D@HAMRq)VfJ<OBWY>=@
zr1^Q)WMkX<N5g$DoCxX9I`&YJl%h;W&6E3<MUp13$&%t;<bI{O&fQQ?60>UEAXcxg
zz+9|*lFNkqgwrAYj*<8N&$(|nT%P-9mPT}h@i~3wNMGt?h%-d0cM3AOQK71pjqkp`
zY1q@RaKEOX(=UblULFqVvZIa9gbX)eAJCjNgUAG$<8WVc)mh9WsiJ0*D}0U>uQ_tF
zz<?HEWX<r|&41AHHp~QLi~5Blea}A~syccb2L)3Y`^){KmfG0S%pBP_BM&B1|B!w@
z+?RPiq`&M)^&Zh}w`)U$brQWdocljp=IV_WB*xUn8;faWHLF4A7;^JdE~e>I;l2x`
z=u?KGYyjd-Yvtg*#Ql<`TdO13?Iyf1&*!`eJ$l4S`ee9og0k%n&X$4krscl4N2VtJ
zpe&7$*j;@h+;@>O>}#DNH9lXS|70I|vNOeZKkvQPrJoD;J^y^Do-w?vHw*7)?ia1>
z%t2Knx`T9uCX}Xe6}xQc9q0oOdZat~SG(pL<vKlRJa#P1E%r?u_g1`fAh_L_oeSJA
zG}nMWt6vNE6)%MJSB;v~@9ERwzLO_I^(h0d$18b0a{sp<t|y4h(mPnShg+;aaJ%h*
zE`dfh>ynoBbYs>at{|*Rg@6R2q~;a5!bV>Kq4{!YO<uPHNYo(j9XFrijREzbPp4RH
zQy-^x^ey0YSljSTYj!RhF3mbsJ*J=3$H=WwcCK4<V%pBgxM2A-vN9)}`#-EIpl+tY
zuRC#C$OzEMacEX{G`z9RI<}#YhWo}zOLvggn1l13`^U{D<^)>mw9Y<8lQ|ms9zPzc
z_ZZgaBy~P;Kkv6ZC%~{fS-+-VF}y?VrXhe07_^<Nv_&7hpSj=gi}8k;k~~ueui+#z
z>4nrhC7w&p)2PKBxsY{RBA5efauG)US#77%IQaJrqXNcIu}|uk!*Bn5Hl(*ZUf$Ev
zSzGmd<$k@n>OH|`%Vs@_=c3UZSPyUh{{LmJ_b+?D?3>Z!3kZM!2!H?xfB*=900@8p
z2!H?xfWSM9Ku>41?U}Za_4I#tXEf@4{-1l|9abEu3j!bj0w4eaAOHd&00JNY0w4ea
zAn-s0*!h3mkuPw^ezNC{p3$RUIePDCXV(|IE<}GY`f<997YKj=2!H?xfB*=*{RH%Z
zdL_cWs!v3FMiZjQS4%Q2r{Im}B>A<nLjM;;NjXE0pB5xuD2euk3atPu8P7Rek4rW`
zziTGV8cDNmeDk?kGmt$vX@t{*x}F%Z3k_#04WA#Mny#0_T;en-nlV<|S(@;sY6YpZ
zu5Anr8PBpVB^NWPrQ}S26#eOXDZ-WY<9igC9t!mq*L<*eucA^S-^0OfA=&fbgZ7i~
zfyL9?>%|CnQ6I7tbF07w)BI$r#n_F~B7Zl+9%|HHs23vK=xb+54Rc|X!<nr>v@|dm
zBcv`Hw392T<@7S2b4q)VMF|X0^8?4NmJYDzjZM30P80Azz4Zvb2ZEK%3b0my?Y=fM
zC^WDpmHK9wlOzIIl-Ctnamq{EnxvG30xuLKMJoy=VO>&GUMusoc+FT&pi29SYb_ZJ
zu<KXquS7V#H^y=`<l8Nkch`!kX8ycjto2lRMXE@o9>tWm>D|<T)*<A}qU3%H$ShTt
z!`uSPOCX<7QRJea?C_h?4l9zoj*P9E+f$bQvHC`Y6Kd_2G@l3Zu7t^&YG>7axnkR4
zv(s^NFR|?X%a*(nOgGdJo%thgp43>(H1xs9Hy2pr`gnak!o8%cdzPUj6eWID$ZxiE
zB@G(+3t%i}U^4cry_+CV@!D*CEyAg<MM?3h=~Wx@=N$=rzFeplORS1Z)dDSNDM*Rg
zbaII|ok3%|#LNsoy)c`Zo9DG+CF_d0kyx63F|l-Ncq$%qmH@GJlNY)&*V*D|j|CZZ
z$S)_;4KgmeYF+5sO^cbOh3mE&jtaXcSxU|)=8|?IzwYS;PsRZaqpSD7a2>tB%k}<Y
z?|<n1x4pk>|M&KH^w0nVKmY_l00ck)1V8`;KmY_l00cl_PXZn7J#7*87G`fp*jpzh
zda8roj<wTUG(>M5?Du~>|IZ-c1p*)d0w4eaAOHd&00JNY0w4eak1heW{~zZ6k8b^<
zWFP<nAOHd&00JNY0w4eaAOHd&zzE>?f7k&CfB*=900@8p2!H?xfB*=900=z#1laul
zX>Nz>?LW5J^RIeJNAul()-84YMc0wYFGa3}e<FOb^S^b@b*SxMZ~sK->un#TsE6~S
zeSA`ETbpUx$idpS#oR!K9gr*v>c(nWP{gcxG&>#G%uW?$M;Y=%{MJazy(o4`Xl_8!
zB7$=sV$eQ5_2v4D5pMZ2Y|Eh~$x_x4#G5M8W)n<BSGm9)XPi|2f>HL~vs=CAUV2Zu
zbW<N>^O>TQRV7W6OY5q=agDW`j3yN;1=?37yCo<x+a{wif{@Qks+!fxn^NGJujX01
z+0~tZvovYr8*`%$(I4*L$tKIrGG}{fq*Jt&g}2alVt)iHwX9~P64TgoYN(yq*{UPn
z6}Xj1u6`xVU6iR(XOwcKp&E@tOpU!boO-l(4RUM2XPmi!-8J-Q>X)OO-lK;p7dnT#
zV(nAKj;C%R#y&dM4YrNm(8hCEwHw2yN1%Wp<*M~$nA>E#{&+OoPwVq4vPb-&eDxdk
z8G`bpexI`THT{EN8E)QU>Vd!*hhDcx!MuWVK(8BzHNHTLBA+&WP@iU(qj`fMH)DT2
z8$N)Y>)Im1u0E}ns9z$V_S#ca2g;s4%~sB73#^Up)gIm^pXNCkSdrEgN!`e@Ev=e8
zlj9#o<Hqh>yE?(Z`YzQIVeXdWL1^PS+8)d5x!x>)^0CA6C&S)m{Gcr_s9jg<FC569
z_!K#SKcNlaJ{sW`3{&2#bA}-Y8nQ0eFGjdTt?GJY>r`#y0Bzd%ZLh8LywQ!(4BH(q
zHJ?siq5ZbJ<C(Yi9PgmvT)F;{Fn5J@K5Q>I&!O1%5YF5-OeoM6cxJ&Hj?VUM6qai`
z?w$`C1{|qhAPdz`Sb3TQT~E&H8Dmd9i4Aa@`!i3(tQ*f9&w9d`zxM<OR`i|vd8%i<
zv$Z07YT$G`4Oz46fqB-#^{EIqUH5Fz=t*i3-rBWgdg2?#W$4g!*t@6j&6_N0g0e0d
z+kZ9;eraDgsbF98bRGe_GP<E&i#|mITq-;5w^yzzd7}<<d@K-!Sf8Xl4|~{Fo?i2!
z4NR4t?1rprW%{d<s#)6Fkj&B8zOJfDN|q)*>=b*pm))woE|;|B<ZN;}&Buc*Tz^-6
zg6;0dcIGr~+z@0AWoQV>H)rToC)XUD<Gn`TYLiQ+M;#7q1*7$I5iYB5`i#$)*BPs<
zL*`Y&+H$7nOJ_2(5@(&nW)o?SX^f0as#I-m^uRjPZ`a2o++yttscm&H%@X%z#m&hM
z=qX)|o9*0!;5uJ>Iutw!!t?)!b2|f4KmY_l00ck)1V8`;KmY_l00cnbVJ3j*{~zYs
zfo>oG0w4eaAOHd&00JNY0w4eaAaFPY@caMaaD)^P009sH0T2KI5C8!X009sH0T6hY
z31I&JFxL)r0|5{K0T2KI5C8!X009sH0T2Lz!y$nA|KV_i6c7Lb5C8!X009sH0T2KI
z5C8!Xc$f)b{r|&UJJ1aTKmY_l00ck)1V8`;KmY_l00a()0OtRP!x2(I00ck)1V8`;
zKmY_l00ck)1VG?nCV=PvALiPDZXf^xAOHd&00JNY0w4eaAOHd&a5w}!^Z$QM5`T;S
z|INdZ3@IQ00w4eaAOHd&00JNY0w4eaAOHdn3jxpk|6x%HdVl~3fB*=900@8p2!H?x
zfB*=900<m10k;1C{cZ2#dOvh5+w+S(r;dI{_Z!{YU4Pj1{n7Tw>*3!BfB4AvA6Yrl
z-ud&LpXhvF#}9T)wf|&$KJ<;y-wO@3eSs1_vM>Fqx)kB`-dCeNql#24Z%KT$B;Tn@
zyj&8cZGN>X7c{w)Rmug)d@Bh>iC>l0%8JC7s)d5ETA(x)QP8BBrG@K!YJMiU!sjxJ
z$*d|#VpdR9dA(GWN?JBpSZ*XfGCCBW8j4Ty@rm>C;q&8DJ9RO_E$i*1l^}||nlD!*
zK3^_WizQww^9_Q9f+i`vW@wUY2+Q$_nHhe1VKy^2&*uz5IX<0SN%QlW*;zgy?^o;j
z2$#`2t-OW2CYMWxk+~^pkKE_$t6}a|%*ebbm-uq2u*0vF3!<c)6$N!;wJaziUy+oe
zOkm5Ud|8zEqP(sMOwEAeh;@M~xSdHX3MFA(5-o8aAu2BvBt_#*4OL!|DiYPF=nCF5
zo2&I)m|JAIn6|*LE9Gj%vLZ4TD+i@okf;*&&{^-SXCvGdeSpLzsQyh`WKya!`^Hxa
z!cIX}wdti~BAw(j^Qpy5(rbElglt(^&&pz!(&hLCvH&|E$DhiXVY$H^3yoa}fVWov
zM3~FGOlo8d3pet&>~KpcR3+7`P3HOx)5gw%Urwf3f{PVJE@^A02A&rO24yiYqiX%-
z2$#@%tO93JGaiex1^VpVzUVh_qnxIBFIXl%lf0bB%%=Hxkg+G~9}jacuxeYnl$f>*
zfH&H>X;fup&)T|Oe<{q}c_0IG40cc9$@=Xucf~8*wSDyg>~pDpE6i;kr-GV!Gj~$b
z$SbKH&_zj<tHsqqd6n9T!q4q2FU~F}XOq)JFUUNm>nmYyjtNyIjjssW1`}2=nj$ld
zHJN;);g@!kqDSgCBiu!Og5<a<cv;*&Z_3=1c3K)bOOP!}nviAL|Nre>>u(#`b*JdH
zX*+AA3v?kQ?KXqi4c5YvyCfw_qS7rYzc$L+l4E(-`>;z7IhWGFL(c5X(6+1y!K|7h
zpZX{CLs1mzmwqi;v>%EB>8Cbnn?BHH)1>bQP_$^#q5+EZ+&d2r$)Ti?NKqnxf*<0{
zaPFPoy>srl=lsq&l}!>66}sM=?BPj1+r#rT;rSJ@^U!@|_uccUvTIXcY8Dp);vqx}
z-jnz}xx0MjCcSYtq*a<{*WX)Nzq>9yjJ9_%(x5MJ{TX>ND~S&D9u==tARvRQ<NNu>
z?mHJ!ZpJOgf@bmM#*><DT%@{UJAv@Cm}6;-8uXvXU$$1Ry;l)WVJflrXyfWZ8h=r2
zm$!HCo=?5)emb_pG;^AY)3Ov@G9T?d*zIu?0*&{!b9iH0-<{rFKc9NzYqidSJ=mDi
z{k`Gr9vd3TNOwBq4j<*dAog4tyo?W}D>v7!h)KoQXMPi!+Z!?M)|A>NtMKoUZW4Fi
zd%Y{FW}Aks@zpWrDsQo(wSi0SyDPV@cQoz20sYR}+A~6aW%cg$x31sm_+CS6u!^%~
z(&^48jUAe|cJE|TZ*vjnzi}$7D=nRtW4<VB%?*Ff9$?*ke)r3n)K|Ysx2h_&M>I8b
zi8V}TOR_g<J_nMnt}$J;r6(HAx}YtiE}5;WuGRcSEYvkB-eIwriMzb{s(0j7>E0Kw
z-?{GH`ps8gjafdwvU~e{D(AjI{Sj(hq!S%$S$AWdYno`a3rQ(h)xyriH1B?TckO&?
z&7Gz8Ri-l%W`<>MDoxUC{HF?Awy%+#r^WlmowZw1cr<u;?@8N-X_*xRzOpJ^_C-5C
z)cNI^xGsHmca?W(#fQY<GsUL92`#h2np}xh{een=QT%;+XJ?38tTy2O`0lOqsTKFC
z-#ew*G#s&-*@y31%uN#a>nC<^3TgiWb?gp-L|YI~-ER+xH=tY~)U^%1N$5T$<)p{O
z<(qfuUG)~Phuv5BUw$avxxTtgqTueD)N@VTZpFR2`!;oN%H5#u`R}l&vhrO}pH?bL
zt;TG-;uw!uQ}nH`*gwb}yoIUHm3)1OHrHqEe|Psw^cXL=JKc}LSH(WJ4ixP6Y5pQ`
zsCyUI_B9=Y6?=<)pC?q_+jr&4Z>)rc|9@-^7<e!s00e*l5C8%|00;m9AOHk_01yBI
zAp)1q{Z#6O)N<-#?ZUS%Tsr@A7k_Z}H_m3C`<3TDd*+wYf1aM6Qc}yA?`Ag7{UH2G
z=Pw`sr2AfG`ZJ$N|NINW?&P@^ynj<a@GTv_RmnRlOy=C_7}$qz-&wh}eCK`X?d$JL
z%kSJ>TUn*A-MYR?Ub?{3LB?)w!sxOY(^22n5#B5M4M%gh1EAB+Oy5z-&uVFA)b8yY
z)P1Xbq$&Tz-sbK-Oyx6f(Y=?MR$fcHHDUNVxc9%yU7z}XoE7Zgx8p96u3Ji%qr8Kx
zB&ZDBggE2=wEJ#mdV_lX(ZPD{`Nu1QMbOns>YIn^c=V13=(~4$)+0pBxL4hGGSjcU
zmj3t)Vpeu_sq16vZ#w3w+;v(eAJ2sg68(_<Zr@3prE4!laTOC#9QFFx{w&aHe+=Ol
zhv+slTkf6A^cP-AyPpq7A#iR7ra_vk!J*V&#{n()Cc!0<Z+T|K-~_jWhIT_?Ep*Sh
zU(QTl{ao5Dg^h<^%(m;0yHvx&sqgjeL3mB)SKD4g;Yj7(XuHjPZ>;;Adpk4zIqJzf
zo`?w?yWu@kpL7i1ki4zrQ(enU-+D3aE(k>S^N8~9rM~O;Y_MKb)m_a@7pNzfI>#o&
zPPF}3dyhtVaeELt@t?W)tEu11Oy_dxug?hN+XiLNOx^uC^(P(UAa|eVZ&E~%M+JXW
zaJRn}k;IXuU~i!>+(r)f4%#IA-81foG>t#gF^vUjs3O(}nvTL#O)dG0*5CI>E&h8x
zeBvYr&|7!1fsa?bO&_@*WTqFW7caF3%)1+%_TNgs{lD9<{q{GyTjot=gZFsmtM2=m
z>Fd<yYwhXkwb_2=&Zhs?Z#p_z`+TFj>GmxnJ#b%hZ)T<+<kIfF@Uh#2+v)0vdSSY+
zN&Q7%Ou8@iJyjh(B+=Kn$=K!kaeu~rJ2Ty&?h?O@IDSc;JKcRr{b?_r-PZ*D_2zWM
zm&C`CuY0lm4n=kQcKrGo_s85XWv1VzCz#zAaQZ^~+um5D{wy)hdv;Uw>1W-QbJOd&
zbV~Hd{z%sxR#94xAwEZ**q;xDo?d<D`Fp7^2s!i%pE{r%I_JK3PAIA$TkfsQ^ouX1
zzji%bJcMxkOaD;=-P9wCfXvZD^*9WfBb7SyYnkcKe?I;3J%Kurjx=Yh!XDEv(ce>l
z+&K!-b0kVHKIAXKzA9J8KEb~Q|6-5kFWQf#pU}1-)bWG%!X;_uEAEZV^zzGT_j93K
zs2P^UJU&~+Hd<DVaVBNtXzGu8U;fDH$d^1(8ac-ok@h_w&TP2XGt;xw{F&J1yW0Ks
z-t7i2?8B}G#k)H1UOP9v@^Zg-HS(28A@U*hE%%vwH8cJ4%jw;+81Tr+$ginC>>TdM
zDKXljZ#|<HfueUv9mU#Uj>ls4zUgn-h<?rD*iSLLzldt#ktBktL$}IGQ7z@Nw-3!m
zhew*3ap&DDw9XaMt|mr^@5+cNZf#XIiHd8`|2Flr(z2PwDYxD!df47;Qh(qh)Jxp$
zy}0*=-c;(nlFul=by}Xo_8-om{$Hm#1D}8Z5C8%|00;m9AOHk_01yBIKmZ7gBLd+6
zkK?G~nScNg00KY&2mk>f00e*l5C8%|00?v<fc<|b9{2<VfB+Bx0zd!=00AHX1b_e#
z00KZ@91#Hje;h{@&jbX301yBIKmZ5;0U!VbfB+Bx0zjY>0r>wr@xUh_00e*l5C8%|
z00;m9AOHk_01yBI<A?zE|Km8ScqSkK1b_e#00KY&2mk>f00e*l5C8(52w?x;i3dIb
z0U!VbfB+Bx0zd!=00AHX1b_e#7)Jym@&Dha=lkY3J|mt92mk>f00e*l5C8%|00;m9
zAOHk_z@#G(+5b;E6nK6h00e*l5C8%|00;m9AOHk_01yBIV~YUz|6@C}crYLU1b_e#
z00KY&2mk>f00e*l5C8%p0@(kDP~aCJ00e*l5C8%|00;m9AOHk_01yBIV~YUx|6@C}
zcrYLU1b_e#00KY&2mk>f00e*l5C8%l2pEr<Woat2<)sfF>}2H)`f%HzPuudQytJ5=
zpU6vvVpe`CFXglH7X3>XX(}De%7*D^MpIm?vklgyquX*#(QEu>vaZ<9s^P4*bX|Us
zl~v6)b!F?8p)z?%c9g2l<g84GYOKMU4jro+j$<_FpuK54x#{1~v098;bPPjxG?O>b
z@a|x1%?)u2+pzezn?{q#JN#0`+F%av!nUj`j$#`vt43{Z%g&a`+cqtu!JJLjvZ<{W
ze->FO&d(K;h1uL<sa(j-&COSHWp%!sTUeYgvub^AcClQR>E2A$9K-UTp~gEx$DW!N
z-NJ4)E?bJjF4v7MW>svuPT61;!>p(+I!-`UYDTN+eE3SG!mq1nO@~>J75%}?j8xr{
zyzgZt!^}$TsbeWKbZ@O`I|>J<uIM()${x%&46C8=XWrfxjqU76;(y6+RLV))+lEPp
zx3|Tuc6R7qmeSmy$LE032UcI<556q#(AYSdLr^ZsYbIUiqvU7}W;08pnX!`{Jd$;d
z>FQOZS=ToB(5S4gv~)gdG=mJ?AVkBFHWi&Y4$IYOM%h}8k3+3RkJ{kZC{H!}D$O5e
zG4%sx86i~S1iw=-;|K|QI)3=3Qe`?luK)3NWDY7V$DqSi#ah<2jixs<Y2vGv@?_oF
z5-8~!&9tms*K~c&RBD>T-wOWbD5^$1@!J#gF)MG<yLE#m9PexJlK6ue&T0SiW9^yW
zdwN0|d1i^1hcF37pBi4_k7^pam{If$yfcbL-QmC7(^QAvT6#52gQnNI_KZfgL@yM*
zl<b;jJC;^$@u|pr<Bzj{18S1LcDMQWs72Wa!ZwvAe}mhdpb1%fMDHHI#fD|H=$)il
zlnq`>?=_@Zv}rNAp|09KY~gSk^}2XZ8;WVtedx{~@+lZz&u{F%<4V;w^p?ZXwjE~j
zOT+0wjWgDyGxQ3z=rNnrOFmKk*+Vm2ZJ%7s?{MMPYHIwoG3ZY8DS_zY5-=kB=o4h$
z{<6>3jVGr|0IpMyB}hR?!3mIpNoAm?{w<Q=<SNrFL|}*dw@It3K`KC_@==R4Hce-j
z^RJh((gD=MYOD5$_16n^mi5%TsBSnT(c-p`c4(IJLK7T-jyMDba1T{F7R&N*vE+JM
zsA%m6Y<V!iUKxsJoebE*>-iG|Y(sfEkSTk$Yg;2-^Y>u%*2L^@NdKe_5m#7AJETkO
z+9qo#?=s64y7DY-gKb~=R~73KzljnmuB<_uSjX3hbrJ(yp(QEFH&9%ctqbj+_NPyB
zvw6O~r5y&}u0G*9D8I<m=>6v#9{=+*zLORG2=-++_?A*y_qLRD2XE6_&$D8_v{=nm
zi{)Z&uC!3gEvjmrcCJ;mzEGRb=VwcFF>Ua@#~`is31g4eX@ktQV0nid0*0b4C$kg~
z`klrB0U~SzunoXA5N!Qn8xXdHF|Z9xth9qQU_a|JY@%&DKWw6;93G=hbhy@l@fUir
zdsgiK59LaW#S;7fqx&dhfW`iwBp->V3E2OWTd-&U|1V?hlaHD2Fv=d1-w=x>gz`{?
za{l%5i9)tKD5LDi{Rrc^++9m%?jFJ@jM>(;06vdT!8r|lo}_zMpUo4vvghE+K8h>5
z$F2u|Hj&Vx)X;>g)8GJ|RHycy`qQZ3oeU*vMCcBRIMAs{t$TF9eb@u1R-GdE|G$lj
zQJah+fM5ay6YLR|F=o3DY(7rW<^#b5#}i1^qgb5yXewWO;EX>n@C+eU2&p0_={SZ|
zB{ji#>R(`iK`_Aq3<e+*fJ~tDkK{cG!32BSU80fDhc*!=vuI6*$>q?%a{m9ZMM%S?
z5=1qh2A9f}#39n&ie8xC@!|t1>Kq}``-e!=yIb9In0@f95C;hS8+=*_>F%b{+SoK(
zJcR;wGlxhJ0r7X5#GlG);<Y?nnut&c&O-=xI6`?aw1`Fy)8jaXqv)OpjAX$%NOW@_
zAxHA*0EuoyW)w)WL#4WbBs)!#%w&}Os_BT#H>8>+$a&)_bj3Mep<93~>yc##$@fKx
zxZb=k>xS{DW!{(l++K8KQ*-XiJA3PWGtwK<<pX4p5T@{>HT2p+GF9x^PD8Wp{h#Ov
zqW#cm&AzBP(jA^KD_(i+VE>nRQn>Mq|EDck5C5O`|0g@GUMzPCrcjtdVG2!@u?D8l
zu5_FI(<&vhF%O4$wq}ki=j_<*%^=a7A`(qf_c0j!d*&ies5JFZYYy$6+b?n2gk+=}
z9H5ETqA^NKx2Gq0RPauQ4mBclhoeZXABe%<VF4K1{Xem+I`{vVF8z8ceYTmp@XKeL
z^dEkJ01yBIKwu~cd}HR!r&3cF&QHxwUHbM*>2xt)oXzFSx%{G(Usx(EEahi^MD;A1
zO>&90>#BJ9WVhvNOVgbdD%?O>Tw<%5Q10ov(o|^m^0LF`a?)GgAt{&Rer(nhHCOxs
zAf96B6$8;~M)RsybVIf_s>)?b30y1`=${f<S_`}oNmJp;Y~|&4@1*;!e7-m@E$dXL
zf*Z|!T8OIHn^i-xRLV)fTcwIA(RS+^FGle`)vOSCXm1%!Ua`X4tecy6gEB94fM*$}
zl(kf!;*nn?Lf0O%Rmz(iB*i2mHRF{jhEV7Q5k*MFH{$XnB!vFq8&heL8RySmAer&4
zwQiA7TFTEuWORuPUIzwJ!$wgfs)|JG&v+CyB3bZ{sZ{z->VhW=zTW869`ePd**S=U
zLyLmrG62kPaz!a|{(r%l%Tf6*gG$PU8eV;#)s*?-LQYW^=W}zj3;A3%znIUJtMlc>
zLbWhot<BQKVgCP~{C}5dT1`#8oT5MYiAUgf?mR~e@RU&6f7nQ;&vd?bh4NB<5$|2s
zdk3vu!}`ms5{}LKdjhJYAO#9ipdbYb>ZzihYInJ)iK!*s<C#SYl&<>RFx$dxJ1Vnn
zQVyFJUo)x{D74-ESH9)-Yt4(Wj6`Ik;qWy;e62CB=zLIt9i`~UxfFd;p#kD6h_CVf
zf0N40GG}wZ$@pTK#Qkei^k}GuW)de_i$Hv>@KDqWRjp@Mq-^YfVqD4~z77%H;V4oE
zBfg&6)YS+hl9wo+mvgDuHR6a_TD8SVW4u7FPsAeP)TG(qk+o{T%YXme1Ie;hWhE~%
zsK!PS|G!W2st)J>7{~upotrNHKkfggo}W(9AN&9TAOHk_01yBIKmZ5;0U$7$2>kvB
zXP*x^ukJ5yr_VgsNqQ|7mkLFYUO{@*?3+rJ=@b`8G;`%CNUx{J;6|eDpSgG&`1m0`
z>`~LhHZ(Hh5<A4%v>1gi=_-Y|ZU;A2s<xrG93BN?JB+gQ52k=4ET<>baykloDUopY
zF`2?|q*}_8b!SWGQSsc<LhoZ;)Acn|scFs@h3R`C?W#t}mv8aJ!Ar8J`;i5M`G5@O
zQHVk0{T~X81bP2y0asaFY3Y<IgMRnpiNaOgBsZ?(uv|@7Y+I}Gz^Pixc8mtUm}}No
zNsKUysUI+lqDSkBZZl8i7I`~-0pqZzygm!Reh7*=BwC<@9kzheC?p#4|I5Q+#`SlN
zt1Tj95f`JfXS#gk|93(`rn3#!RBsqoL!pCmpVBVK|8MXRNRseghNpU-74xOVYOY!=
z7jtu^g<5V=RqMI3T2<={wfTI0wnP_)@&8nfsMG&X+y8$z9KOnUc`|N_VUR_JC|ZG>
zz`_+SQ?}OJAdQ!<HY`3+o*?9xD%J+$OWP1qG4TvihUG7DsFS)hBzcyQ>3is+lZOsC
znG}-m2#}o&e(&)EI~-~+Vx*27BZa-pY23>oSc>P_^proPI6qfV7G`scrE+19yby!a
zo(31&gDL?cmC4`$MX(ePf9fhNIx2W4Lzx;8y2H_^)(=!AVC<Y;UjD!Psj2&%34otL
zBJjzJ=h`Hy-~1^3+}SP?)xuH<89R~h8@?KZEgD4QAtP$y+RS<S63>!PiCqn|T{6J8
z;*yt9(QDi$i`0Qg9k|C9am<ot!M1Z$+s<(k50lG2KLt)8{3zKlgdg|Gv8Sd*KDj=A
z>7M->atDrg?m&pa(<BBDrvM%yc_8g||B3H+K^Hr*nNvnV7egEX;s9hva|Ev)aR3TD
z%_A@3FbXk<H~_=}STzdT3A3pkfkX`ZN`y=5hb}zUp84!wZo_QaR6}%?3ipw<(!YSm
zRITus2h-36o0_(^envY{-H+YVR9-ohY@en<c`w(sXEa==BPHG_BpTuXhQyogV{eI)
z4G{+rO-d_@+0Be1uJwzWBi-S`gsf$Ll1xWyFcLl*qYe0jtZcD5kGUpUb0YHq3g|)#
zSW^D~LJa@^KmBC$Rm1^s3Q>X#${?}zIMtBsTa-Ib60B10pG!Tuf*cdFlR;fOeqe{g
zT0<NFtTPj7ok{8}LD3WxO+nF=t{`i3QsOClm=92<5~`_%*YjDakDX&g=nhAtny@$k
zkN^KlYU-6iGT+DRUZ32`w0Zo$rKHcE>*Vpz7MAjPBts2SMNrl=&PXOw9vSN&aq=-)
z)pYGKTQ#19wtZ7+GH&W0Lb3?>THtF@q7`6A;cH3GY4vKs29#Ft`8(6R^ie&ccAOK>
zW?eJmoepCcREUJaZPI}6uqGEb<?(%ToBOG0*$XQB+O!UKI=yX2sXd|`OPqJGVHqux
zE?%a2(}*MxrJJ?+`F;k{FhP@1ugg1hMdK98K=Ataw9@nuf<61M6E6w<O%}wV!~M{S
z%X@>@e;UPMQj@i(5FUZwZ|cfckQP>{illvXD0G-qX_x*Ve5VfnKluL-NMmR<HJ;J&
z(ER@ex(DC+69W(7c>9Or(cUFl1=UgTrNIAh3rwYj<$7{<FGuA62f~mt?m`%L2}8kQ
ztWa)yK_7DrN6|e|I3rcJBrk{85riG7)RxkuEXg8^v0!wr(6U6^XwL5pBAWzTJeC9s
zdkiX@gpP_avd0{m|6lSIJYo?;rgh1*nCyc+#_4UteDl7n8^)uSd0)<YX|U-C>Hwe)
z0AH9I%+zX+7^U#orZ<g$er8ZvDN$Q7ShkMF|6h#Z|Nl4m{{*`8;VYF2`2Qk*4C(+3
zDNrmPSn&UclE`{5%35@3J{~UMaM)_#|A*#mSZ5~5x17|O;(M0M{tjjiI~dL<2s6gT
zE=;9kVEiJ45_JHoRIY5koLlHh#+S5g#U62#Jq(HCS@X-25llHEbcdr+`P3BYnqR?j
z%sDC^|390W%8nOje;7~m$@TM*IDjvw&t*E}0Ol5#^0SBoKpX&SFH`B;Na8GTV3F6K
zJ75P;u&fUSHfql~@$6T`0l-!>Nw%8gu~3KuKpemhXgHwZj3W&vsp)!XaR9VO{x7~y
z*6gc>9%d|a*i(m$9tx_Goy@9aqljikNB~L#gv42d1Rx~9n#iPc2nj$*0F(sqlJJv?
zSl4uY%~WcdvqeexNlo3b6jftQ#}7Jfa`I}H*v%@tK@*1?guMiGp;qq%z5<j4po9$?
z*~|&Wc4`NXn4kASsoI9#ayXv0!>Cv+&zw}_YCG9gH!Q}Qbmp<5w}>v>q$&!0_Z*zI
znA$$MnBR%SdLkrXNKDUz+Nlu|KxF{hL0?0U`kL_k|K%9||NnZKWt_Ma0VoN8kN|`P
zY?(X==LqGW21sR#dOE>k8(<dO;|F#)1U7^OAS7TStusmeDhLVK)5(mGfC1b5V*%|S
z`!^z1HA(^`jRNNOt<a*iUpxy!0upYh5}`XBjcS}j0tEj*^}8wl=T!cDFMpv;%Kz<G
z)0y)<r2Hb1^WO~&?POn0l$@Vz$hu<MWI+z9@@`m~dfOla^tSBP-le4eMT)0<Dva0U
z2?$b{Q?7|H%YQWQl9%?0EYg1cO^eC`fb1oM>=jP2G0M4=ocrq0y$XJyGU@=J4!{u%
zkEn%0%S^EL!O1)tCv$Qc=$mE&AxNc!kZ&%Mt)Y)|xyNE~;v=E_omE>GjMicY{~s#h
zII4t6?bMz^7-X)Q9h@wr{r>;Z-~SVp0c^wK$=?NMi<gga6l;U=QZ+-U(3aF4+CLG^
zK%92_+3ZSrDJN}ji_D8VJCgWc^0OC<Glofrx3?*9!cum2NSgIaKRg7HmK4kW2Iqlb
znaEr>DYoVjAUheEXO17(;aHgAvxm<fK6|cgaBgy^g_)-;CVDyT0m;G)pFMo`vZ^?W
zZL|g!i7_}8s%Hm&OborjD5zJ4;)Y3#ZhsARRPauQGBqM}hoe#XmZeEa<9FzIeeUyz
znQ0u&Oo-VZFn_O))t&aBDB~BKz!%O(I3Gt|gd8#Zh}j?5P7x9f5^dxX4Ke#e3dQWN
z;HiEx_=;duAZ8yi`%OVwK9-t&Ju&;81ouK|DL>zNsFYutEB)w4^*k%)ON-T9wOB6Z
z=1L2-+@h-1soF+WtuNH(^ZD75$N&FMDn)<B<>z}ZU5tbTOr_6Xh=XTaR+>#JJ<PNG
z2{yZ~*~EY#x<j?4>COrzV<GZwtgV^y+|-q(LXQ3xC)3Nhob;A=NXq57ABQzXG6lc>
zu;&wZtQI5kea&c!Y>V_YYon@M&S#}!p+NtXh~-e2p?fx!1`ju1Zud^Q-^%BU^U|`;
zETUx48tg@+tBSo@H55yw!p6K+Rr8WK`0ds;yQb*xGsRjbHV0+pX*$H%CproB-`qq<
zfUL5*($YD}B}|qRCfqR<ojDH6)pW(SwHhDQT8l^@B8?GGj(L?59WhF+^8vGr(0(LN
zAeko3-x4J%qS?o;(OYF{)fSRz%B`lxDOSnpum}m*E6>A0)waF`UIn2mD;cI=k(mqO
zeK=tQVS_y&L9RK(zFU$}5&$IuP!iw(%nyVFSblcJ1I0!4_;e8x&{bs+COw$+MsCs@
LO-KOk_kZ~RJb2Z*

literal 0
HcmV?d00001

diff --git a/etc/prometheus.yml b/etc/prometheus.yml
new file mode 100644
index 00000000..5cfb861e
--- /dev/null
+++ b/etc/prometheus.yml
@@ -0,0 +1,16 @@
+global:
+  scrape_interval: 1s
+  evaluation_interval: 1s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node'
+    static_configs:
+      - targets: ['node-exporter:9100']
+
+  - job_name: 'foyer-storage-bench'
+    static_configs:
+      - targets: ['host.docker.internal:19970']
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 4ec0d77f..ac51628f 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.1", optional = true }
@@ -16,6 +17,7 @@ foyer-storage = { path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
+hyper = { version = "0.14", features = ["server", "http1", "tcp"] }
 itertools = "0.11.0"
 libc = "0.2"
 nix = { version = "0.27", features = ["fs", "mman"] }
@@ -23,6 +25,7 @@ opentelemetry = { version = "0.20", features = ["rt-tokio"], optional = true }
 opentelemetry-otlp = { version = "0.13.0", optional = true }
 opentelemetry-semantic-conventions = { version = "0.12", optional = true }
 parking_lot = "0.12"
+prometheus = "0.13"
 rand = "0.8.5"
 rand_mt = "4.2.1"
 tempfile = "3"
diff --git a/foyer-storage-bench/src/export.rs b/foyer-storage-bench/src/export.rs
new file mode 100644
index 00000000..7c21e071
--- /dev/null
+++ b/foyer-storage-bench/src/export.rs
@@ -0,0 +1,53 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::net::SocketAddr;
+
+use foyer_storage::metrics::get_metrics_registry;
+use hyper::{
+    header::CONTENT_TYPE,
+    service::{make_service_fn, service_fn},
+    Body, Error, Request, Response, Server,
+};
+use prometheus::{Encoder, TextEncoder};
+
+pub struct MetricsExporter;
+
+impl MetricsExporter {
+    pub fn init(addr: SocketAddr) {
+        tokio::spawn(async move {
+            tracing::info!("Prometheus service is set up on http://{}", addr);
+            if let Err(e) = Server::bind(&addr)
+                .serve(make_service_fn(|_| async move {
+                    Ok::<_, Error>(service_fn(Self::serve))
+                }))
+                .await
+            {
+                tracing::error!("Prometheus service error: {}", e);
+            }
+        });
+    }
+
+    async fn serve(_request: Request<Body>) -> anyhow::Result<Response<Body>> {
+        let encoder = TextEncoder::new();
+        let mut buffer = Vec::with_capacity(4096);
+        let metrics = get_metrics_registry().gather();
+        encoder.encode(&metrics, &mut buffer)?;
+        let response = Response::builder()
+            .status(200)
+            .header(CONTENT_TYPE, encoder.format_type())
+            .body(Body::from(buffer))?;
+        Ok(response)
+    }
+}
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 83ea28de..a84a58c5 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -16,6 +16,7 @@
 #![feature(lint_reasons)]
 
 mod analyze;
+mod export;
 mod rate;
 mod utils;
 
@@ -44,6 +45,7 @@ use futures::future::join_all;
 use itertools::Itertools;
 use rand::{rngs::StdRng, Rng, SeedableRng};
 
+use export::MetricsExporter;
 use rate::RateLimiter;
 use tokio::sync::broadcast;
 use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
@@ -240,6 +242,8 @@ async fn main() {
     let args = Args::parse();
     args.verify();
 
+    MetricsExporter::init("0.0.0.0:19970".parse().unwrap());
+
     println!("{:#?}", args);
 
     create_dir_all(&args.dir).unwrap();
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 8a65e575..be6007b7 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -20,7 +20,7 @@ use prometheus::{
     IntGauge, IntGaugeVec, Registry,
 };
 
-pub static REGISTRY: OnceLock<Registry> = OnceLock::new();
+static REGISTRY: OnceLock<Registry> = OnceLock::new();
 
 /// Set metrics registry for `foyer`.
 ///
@@ -31,6 +31,10 @@ pub fn set_metrics_registry(registry: Registry) -> bool {
     REGISTRY.set(registry).is_ok()
 }
 
+pub fn get_metrics_registry() -> &'static Registry {
+    REGISTRY.get_or_init(|| prometheus::default_registry().clone())
+}
+
 /// Multiple foyer instance will share the same global metrics with different label `foyer` name.
 pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
 
@@ -46,7 +50,7 @@ pub struct GlobalMetrics {
 
 impl Default for GlobalMetrics {
     fn default() -> Self {
-        Self::new(REGISTRY.get_or_init(|| prometheus::default_registry().clone()))
+        Self::new(get_metrics_registry())
     }
 }
 
@@ -90,7 +94,7 @@ impl GlobalMetrics {
             "foyer_storage_inner_op_duration",
             "foyer storage inner op duration",
             &["foyer", "op", "extra"],
-            vec![0.000001, 0.00001, 0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
+            vec![0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
             registry,
         )
         .unwrap();
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 8435893e..0eaf1126 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -17,10 +17,10 @@ crossbeam-channel = { version = "0.5" }
 crossbeam-utils = { version = "0.8" }
 either = { version = "1", default-features = false, features = ["use_std"] }
 futures-channel = { version = "0.3", features = ["sink"] }
-futures-core = { version = "0.3" }
 futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
+hyper = { version = "0.14", features = ["full"] }
 itertools = { version = "0.10" }
 lock_api = { version = "0.4", features = ["arc_lock"] }
 nix = { version = "0.27", features = ["fs", "mman", "uio"] }
@@ -31,6 +31,7 @@ regex = { version = "1" }
 regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.7" }
 tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
+tracing = { version = "0.1" }
 tracing-core = { version = "0.1" }
 
 [build-dependencies]
diff --git a/scripts/jaeger.sh b/scripts/jaeger.sh
deleted file mode 100755
index 09c103d1..00000000
--- a/scripts/jaeger.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-container="$(docker ps -a -q -f name=jaeger)"
-
-if [ "$container" ] ; then
-    echo "Stop jaeger..."
-    docker stop "$container" > /dev/null
-    docker rm "$container" > /dev/null
-else
-    echo "Start jaeger..."
-    docker run -d --name jaeger \
-        -p6831:6831/udp \
-        -p6832:6832/udp \
-        -p16686:16686 \
-        -p4317:4317 \
-        jaegertracing/all-in-one:latest \
-        --collector.otlp.enabled=true \
-        > /dev/null
-    echo "Browser jaeger via http://localhost:16686"
-fi
\ No newline at end of file
diff --git a/scripts/monitor.sh b/scripts/monitor.sh
new file mode 100755
index 00000000..bab191fc
--- /dev/null
+++ b/scripts/monitor.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+compose=$(docker compose ps -q | wc -l)
+
+cat <<EOF > docker-compose.override.yaml
+version: '3'
+
+services:
+    grafana:
+        user: "${UID}"
+EOF
+
+if [ "$compose" = 0 ] ; then
+    docker compose up -d
+else
+    docker compose down
+fi
\ No newline at end of file

From 0346f6943b0d1eb41d2f7aca0ba621cde9719503 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 8 Sep 2023 22:45:59 +0800
Subject: [PATCH 099/261] chore: support keep prometheus data under .tmp (#128)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .gitignore             |   3 ++-
 Makefile               |   6 +++++-
 docker-compose.yaml    |   1 +
 etc/grafana/grafana.db | Bin 1040384 -> 1040384 bytes
 scripts/monitor.sh     |   3 +++
 5 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 804ff444..ac819a7b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@
 Cargo.lock
 lcov.info
 
-docker-compose.override.yaml
\ No newline at end of file
+docker-compose.override.yaml
+.tmp
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 5ffae04d..07c23e91 100644
--- a/Makefile
+++ b/Makefile
@@ -20,4 +20,8 @@ test:
 	RUST_BACKTRACE=1 cargo test --doc
 	
 monitor:
-	./scripts/monitor.sh
\ No newline at end of file
+	./scripts/monitor.sh
+
+clear:
+	cargo clean
+	rm -rf .tmp
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index d4d1bc7e..37ba2568 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -23,6 +23,7 @@ services:
     restart: unless-stopped
     volumes:
       - ./etc/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./.tmp/prometheus:/prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.path=/prometheus'
diff --git a/etc/grafana/grafana.db b/etc/grafana/grafana.db
index deafa6c1cac348c755ab7e215c08275fc0670db3..65c81c1be572965dd7d1ba43b4dee1d15e145963 100644
GIT binary patch
delta 440
zcmZoTVBc`Seu6Y(<U|=~#>kBc33`ki%?I_`59%=jF%u9o12GE_vjQ>O_JextJ_{MA
zY!)==<exs{KD#vYZ~dm}Tko?guxh3J)-~OJ>puH)epU`yW_!lzd+xJKZomJYUDbhu
ziT5M}-!tBmK#hBO&6=c`*wUMQm{}6@^7C|yQ;Ul;^Yg4y4U7y@EiEk4ED}@FEK<yj
zjSMZ*EKHJ<Q!SFrQq4@K%h+?saatPb8Jbz>85&G?u;*AJV`yS!WMO4!sb_9vZfRs}
zVPp}dRIA?0kn&r<dHORuj(-|Z6&8Bt=Emm6rX~hva1~8l3=9lR{Bs%j&-2d(I&C7q
zks)(BBT24enttA&!<2(kQ`*nRb8J5!&#8BT<+pw#?`FmW|K*o`U=mn#L7}aItF3{%
Rt%0Ylfw!%JZ&?F>0s!k^jTZm_

delta 235
zcmZoTVBc`Seu6Zk`a~ILM)i#e33`mI%?I_`59%=jF%u9o12GE_vjQ>O_JextJ_{MA
zZWc7?;-5bKKD#v2ud~y)-e*@}<xBbXdg}ID_t~HGv$D!E+cQqzbDv#u`~COqst#<7
ze9sv8o&j~;;hX-<j^m%Ifu)s^k(Gg|o~4C}xsjo<fn}6Zt$Hs*%CF~pxfmE282QgL
z@J}~%V41gB!9at5`gwm2Q&u#=_Ve)^+t0^y>Rn*@bykLVGvk5(^2<Ij2`svx(AL1!
U*1+A?z|+>i+t$FhtbsoP02nt@`Tzg`

diff --git a/scripts/monitor.sh b/scripts/monitor.sh
index bab191fc..033d7447 100755
--- a/scripts/monitor.sh
+++ b/scripts/monitor.sh
@@ -8,9 +8,12 @@ version: '3'
 services:
     grafana:
         user: "${UID}"
+    prometheus:
+        user: "${UID}"
 EOF
 
 if [ "$compose" = 0 ] ; then
+    mkdir -p .tmp/prometheus
     docker compose up -d
 else
     docker compose down

From b3d41735f877c5398fe1eba7c627cd783196e993 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 8 Sep 2023 23:31:06 +0800
Subject: [PATCH 100/261] chore: use grafana provisioning (#129)

* chore: use grafana provisioning

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* grafana use default user

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                                      |   1 -
 docker-compose.yaml                           |   7 +-
 etc/grafana/alerting/1/__default__.tmpl       |  53 --
 etc/grafana/dashboards/foyer.json             | 820 ++++++++++++++++++
 etc/grafana/grafana.db                        | Bin 1040384 -> 0 bytes
 etc/{ => grafana}/grafana.ini                 |   0
 etc/grafana/provisioning/dashboards/foyer.yml |   9 +
 .../provisioning/datasources/foyer.yml        |  14 +
 etc/{ => prometheus}/prometheus.yml           |   0
 scripts/monitor.sh                            |   2 -
 10 files changed, 847 insertions(+), 59 deletions(-)
 delete mode 100644 etc/grafana/alerting/1/__default__.tmpl
 create mode 100644 etc/grafana/dashboards/foyer.json
 delete mode 100644 etc/grafana/grafana.db
 rename etc/{ => grafana}/grafana.ini (100%)
 create mode 100644 etc/grafana/provisioning/dashboards/foyer.yml
 create mode 100644 etc/grafana/provisioning/datasources/foyer.yml
 rename etc/{ => prometheus}/prometheus.yml (100%)

diff --git a/Makefile b/Makefile
index 07c23e91..a249a1bc 100644
--- a/Makefile
+++ b/Makefile
@@ -23,5 +23,4 @@ monitor:
 	./scripts/monitor.sh
 
 clear:
-	cargo clean
 	rm -rf .tmp
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 37ba2568..7a46ae9c 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -22,7 +22,7 @@ services:
     container_name: prometheus
     restart: unless-stopped
     volumes:
-      - ./etc/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
       - ./.tmp/prometheus:/prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
@@ -39,8 +39,9 @@ services:
     image: grafana/grafana:latest
     container_name: grafana
     volumes:
-      - ./etc/grafana:/var/lib/grafana
-      - ./etc/grafana.ini:/etc/grafana/grafana.ini
+      - ./etc/grafana/grafana.ini:/etc/grafana/grafana.ini
+      - ./etc/grafana/provisioning:/etc/grafana/provisioning
+      - ./etc/grafana/dashboards:/var/lib/grafana/dashboards
     ports:
       - 3000:3000
     environment:
diff --git a/etc/grafana/alerting/1/__default__.tmpl b/etc/grafana/alerting/1/__default__.tmpl
deleted file mode 100644
index b8633d16..00000000
--- a/etc/grafana/alerting/1/__default__.tmpl
+++ /dev/null
@@ -1,53 +0,0 @@
-
-{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ if gt (.Alerts.Resolved | len) 0 }}, RESOLVED:{{ .Alerts.Resolved | len }}{{ end }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
-
-{{ define "__text_values_list" }}{{ if len .Values }}{{ $first := true }}{{ range $refID, $value := .Values -}}
-{{ if $first }}{{ $first = false }}{{ else }}, {{ end }}{{ $refID }}={{ $value }}{{ end -}}
-{{ else }}[no value]{{ end }}{{ end }}
-
-{{ define "__text_alert_list" }}{{ range . }}
-Value: {{ template "__text_values_list" . }}
-Labels:
-{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
-{{ end }}Annotations:
-{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
-{{ end }}{{ if gt (len .GeneratorURL) 0 }}Source: {{ .GeneratorURL }}
-{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: {{ .SilenceURL }}
-{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: {{ .DashboardURL }}
-{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: {{ .PanelURL }}
-{{ end }}{{ end }}{{ end }}
-
-{{ define "default.title" }}{{ template "__subject" . }}{{ end }}
-
-{{ define "default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing**
-{{ template "__text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}
-
-{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**
-{{ template "__text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }}
-
-
-{{ define "__teams_text_alert_list" }}{{ range . }}
-Value: {{ template "__text_values_list" . }}
-Labels:
-{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
-{{ end }}
-Annotations:
-{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
-{{ end }}
-{{ if gt (len .GeneratorURL) 0 }}Source: [{{ .GeneratorURL }}]({{ .GeneratorURL }})
-
-{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: [{{ .SilenceURL }}]({{ .SilenceURL }})
-
-{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: [{{ .DashboardURL }}]({{ .DashboardURL }})
-
-{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: [{{ .PanelURL }}]({{ .PanelURL }})
-
-{{ end }}
-{{ end }}{{ end }}
-
-
-{{ define "teams.default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing**
-{{ template "__teams_text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}
-
-{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**
-{{ template "__teams_text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }}
diff --git a/etc/grafana/dashboards/foyer.json b/etc/grafana/dashboards/foyer.json
new file mode 100644
index 00000000..eed4edd5
--- /dev/null
+++ b/etc/grafana/dashboards/foyer.json
@@ -0,0 +1,820 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 2,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 8,
+      "panels": [],
+      "title": "Storage",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)",
+          "instant": false,
+          "legendFormat": "{{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Op",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)",
+          "instant": false,
+          "legendFormat": "{{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Slow Op",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "instant": false,
+          "legendFormat": "p50 - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "p90 - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "p99 - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "pmax - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Op Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 9
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "instant": false,
+          "legendFormat": "p50 - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "p90 - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "p99 - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "pmax - {{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Slow Op Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 17
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ",
+          "instant": false,
+          "legendFormat": "{{foyer}} foyer storage - {{op}} {{extra}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Op Thoughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 17
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "sum(foyer_storage_total_bytes) by (foyer) ",
+          "instant": false,
+          "legendFormat": "{{foyer}} foyer storage",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 25
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ",
+          "instant": false,
+          "legendFormat": "{{foyer}} foyer storage",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Hit Ratio",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "foyer",
+  "uid": "f0e2058b-b292-457c-8ddf-9dbdf7c60035",
+  "version": 2,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/etc/grafana/grafana.db b/etc/grafana/grafana.db
deleted file mode 100644
index 65c81c1be572965dd7d1ba43b4dee1d15e145963..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1040384
zcmeFa4UijGe%RTVuWrm2L{Su^<x*@7S4%>i5$A)$;gHMaLNiSY%lY770FjiI0(yX+
znHB*w*k}wnqPR@YNGsd%$GKdWN-nP0&WG)~IG4n}+&Oo4t{l5^cAe|&*cWeHSGF&S
zQ}$KG-gBJnR;;b~{;&IW_iNw*oWW9BiuzAUAn-o_@Av-i|33QTzq+!ZTbjIXm}S+H
zr-Y}3NJRLOEDJ){4M7n8EBsIYB0mgYdc0roZ-n{Y;wzN#O76405ZfR7HIn9cV;{!8
z8T+%?e~A52?BB%x@7TYJ{YLE9hIYGud*H3X@9(a3{ZZG0NLu*47RBv1xE2@V7mh~k
zxLzr0cT@A%b49hbxn`(lF{c&^M!jO?^r9d3{9IZ|W)wNKG_PEj^R3e7<>e(g@8`<P
zC-OmI`NYo4&x-M*M<eeYv(&Yc=BM+&JV&d^Bwt-n!Ugr`^<p0C&nTCbw0t$4T1=*I
z$XApbax$A)PAx&sMP(@?FD=9W*@cDvQ8~Y*nKj+01licgTB*K~mov)sa2`vyN*X7;
zs9Gu~m}#Cb99v7l)Kp6Y$>#xICbg)91guw~Sbl6`{g4=+J`%A-zlRKSBUi6!ru(56
zz2EWX&hetWq;AJ}p`>4&f&gj4?s-Ox&p}1gPDR-jCFiAczUW1-5HI);tuMQjTFLr4
z(I#Va?W8EEd^8ZFa8Qg-ffU`%&2<xdwLjCQwX|4A3%&HDu{1ofv+<M|KX@?m{<Ncj
z5DfnvpaoK;gAmB_$knZ)mU(KW7K^4<t4(;SMKzZOPc{S>bZfgIsAho%HH5vRS91lU
z*cj}LYO}F`HN8~QD;qhjtm>tPf;!jUjUmupurGo3T3Wg4Y7!B-Uh{v)nmg2w%eiS?
z1@c2lm(qwtey}e-aWwM4R~tW-gTVh4ZC8s6i`XtdJ*mS9PHYUcZi=f|ex~7emAGj+
zO`#Eq-M6BSdHi6&(N4cq|I1)lQ~mG`_0<!|GpelJYHK<}s>0Y?R>!F#n?_(wX%MTK
zOlGgAn=G<k*@Ahvm|NTS3=w0{X_<~hxtdn7m^h~JAU;RtgTNSjVrpf>(sE?jwHgAA
zl;nBkQZl=ck;lnY<6H7J$rA6Sd&KymLy_+loem}@nEc-t?p6}iVUM=$wUV)+L))A9
z?xQ!WYOQw1Fkw1#OQG~?Y6-Z9f=snitTlucjB-`2G}@nbTBz5cOUgv*My!On+o{lI
zeO)J$sGo+4Wo>GK&81--TQsO|Q!7<->m_xAm}|c>s%3#XK+2L@vp^fQN=^kZA#H<L
zXjM_KIV+&{<v>Evg)O}Rn(3~|<X4xM7nI~uP+Q{lzP@<kP~^Tpdpk3`!-jklsd6<>
z)<HVgCQ-d}vE~!6_wvehl!&D4ZC@@`Heyn}a^0ZB;5~s1g5?8!@s|!p>b}@S4~PMQ
zf1hqywwK6XQ4lljn4Lti9~KZW;{*!`-#)r?oBwl&7SAkZ{WxzLY?=$)QIaT%Fc}<*
z*aQB2<6v~!=H?~8dzx!#OcrMDvKjT*!)pEjF%;j^_iUfDvbU|L#rTmUk=^s&!o{(L
z1P3YjAgwQ0!Z^VlS`ktzyVz#u1*aLUB3Wb47~hV^T*B^)iSf&T`+~0~FirRiU}n#v
z7fHlZEfuF_Hn$&0G*Kn5d85H5hM-8;hld@@z4N@opCl*$OJVM`lup);Ud2szg%~=2
zf5h*W#MQ38_{o!z8?J1(OE6u6;kaGbDg~_;{@ERpxFPZc8A+p%GqDqii1FjcBfCkb
z%(A{=supaea-gW8A6=pH@}am6)h{T6=vmWr1v2wO>jh{l%s7p98Ti2n`W5>%`273!
zBz*pN_6zX&efCrEdD|vi^&4-)$K@6Hm`uXQkyG$-_z>BfABuedpThsn+ndLJIrfXO
z|2g)LV}C#P_hLU5`x~(ziT!|^oMd^(->!K`66QYWjSfw@!$T$}NG56^JILj&iLsdw
z+|(s!@2Ff)t!7rkE7eXkuH7hayN=k$dVw_v-UQ6-OBfpgLkIQ*oQ{1=R)4`9&Is)<
z`kG@C=O@P}r^d!-$Hvdf<1^>RC(cdJ?0oSVNziuhL8oV(oxjYP;R{BoUarWNAy?|9
z5?LiA7cy|<;o5dbL6Y<H^4#)5c5z9zU{eJ;(|(O`zCU}Fi0G=<s*>SmyTFm=e1B8r
z^XcWQUhU1|b26lrr6intv=KP7%&VFM&>lW03D@id08AE(oJ?{+kC1|OsXw0RI)q`D
zZx+AVegrEmnIq6UPfNlTd&rZAoTWLCUvVb%Q$&VWobf_OGPC3$am1YJ6?V-*Lditb
zupr=RlPw&Ogp_^Qt2|g$?wi71mZQApUwBFsX1v}g>$MsjB)CdOj?2hKPxh@jhh9BK
z=ylTuu#4vXwSdjpu-xWJ{pzqNTs#T2(m_a=Q=?nhgo$0V#>N~|n^sn`skAcB5#8<^
zl7y@epQ;_RLm&e<DEJ|Y$mH9v1}+VX!fXl}GjAGI*(eohOC0suns7#|YdT>5@a7I0
znddZYXKO$bF1~*NN-P-FZJBgru4dHDf(BC(Y1P(bYH3wTXUJ5$94zlWV@MV({yvfX
zfKldd;>ZtYt4O|hi$e0<Rd({tom<E^nES~;&f1=Q@ivd-JDs}7S8t{w-`zcDLS9uC
zl(|f>Cns?qu|&)<FkdS9QhIsOg_YZy;DN98i^Adqg55bXmYo%^Oiopu-g3yuV?dvp
zl6`OJcs8UKZCk0+F?*=g?Lg<)l?WRcFzz(CTaYAS`fK%|tt6AT?D5*?D=pc6u!`H^
zveMHO-U^XwVDn;gA8Q?7!@>t4pN6v$ZZE5%C@kNAK2B?75Hn`UGmS71h;B`=OAI<o
z(=~%<Us?>A)jm;3%|R7KV*6SR7}o~UnKwM1c;j%jw@sn`fZ!MEjsuQWt@Mb(92wkj
z!YJ1(I_%A;yhQawON2)<t}ESb>vQC6Z%6hyq{q7?;ndd-@rHD}O$@abo$OyeiQ}1>
zCb(wEpDMYr-jvB4D1IlKkryJ8klf7$UFI*<$g<QOukHFSl)OWu9F);=uMZpPukUK`
z`RltW`22<4)A0E-yU)SrkL{A(`(N8B!{-m~yaAs-uyYAM-`$yj&+^V;_{{H+ef%5t
zZTP%uUx!b{{t|qivroe3gnbY`U%m$)FXrImS#nG;Fa{s}hlYlKNr(?$iNLP@4`aVG
zycYWhv7d<jN$j2BE3uW>#n{Q%vDmZ2|8@BHhyT&=Zw~*9;rqisH2h=3q#FD~0!RP}
zAOR$R1dsp{Kmter2_S(_D}h7N?o*NADn32B=pX9oJ{kmf^~bvpM{62v^R3!b&vd^O
zS*z<M3$725%cxz$hq_;kR5jDN%wkle;b*&#MVZriEq0*$SOiX!pmf;47j8+hqR<<a
z4t5`kI0Xzo-F+l-YYT3vG)%4cg{QjbqH0MqEx1pzvaWB`=@tG>og~<PHr74Giys*7
zJ`(lrJLwzhK1hB%J=lFxg8QA|-f_4Fv{5O;jd)KDbRU<TQzW<vid@{b4)k{)51I?N
zG?ltyzbZTnjqvU~r(h09-3dvpR1CN%Ol~@`)QzVj-N(Cv(*h_r^8fYsbsv`0s-C;0
zZ9m`JJu3wTRZU|{C-;8C1u0Ov;h>Je?(U<Kf4h&^)qN;R4xtW+q)G>eSubg%%6`Zg
ziT#lP|KT4JKmter2_OL^fCP{L5<mh-00|%gB=E^5@LW&)WYoDq(fRli9G$48O`~SH
z`~PnVv2O|PLpY!8cB1r100|%gB!C2v01`j~NB{{S0VIF~kU&5{>W;g|0YeZLi5(N*
zKm0=iNB{{S0VIF~kN^@u0!RP}AOR%sX(nL55jiAC@smS#_Qe;6qETsRYD$`tB;Mup
z*@-oE;;eRhc4m5fQdK9l)9Yu}$BS!cXNr@vvu7t~XU3<;*C(dWoGEH++FD_JdTn}b
z;!JVobm7dI!bDM>nwp-PoLSRmr)JKaIXy9PX1Z{?pw6B-v!+c!i8C{*w!XS?BVEdr
z)Z+Gf23{a@L;3QTo8QJU%MF@ncw5`7d;R}UgxH^an!6a=js%bZ5<mh-00|%gB!C2v
z01`j~NZ<(~@N_qQ2q3uspIrY3r+@f|1dsp{Kmter2_OL^fCP{L5<mh-00}(71jzM&
zy8j=BZ{#?D{EL4`00|%gB!C2v01`j~NB{{S0VIF~kiZi}fNb~=2{(n|KNuPv{Dr}V
zfo~1`?SYHY8`2+2M&CF4ezNaK@9*}m^&ac_@t)6h|43IFlJ5O)*E}Q%b072u-WQ2a
z0>)!B_(dI?Y3F{?_hHbX>qY8Bnd@n?t#8eJ_{1fCC-Px`xWp;A7sPn4tbwbCdW{pI
zSGIHu980WiyMB;~pGa^#tz1^p@=|tTfw)_CB+Avm=>!wU6duIqj7o!paK~}9RyHgR
zyi^y#b#X?yp7Be5#4b0#_>3fIyZ4~esq67(5A=-Z^Gdx`QrAiv>mS$gl+5_Bx5`y$
zxVfblK>k9(s8=lc)#c>{CAk!Mu0P+Oy*i)FD6V>~DjAuV?SkAa&H4VO${mm5bIS|a
z#ihMvpuT)*7GBNZ=9g1-^HVxA_V7VTxMq7E=bJMz^a!bkxLt3~->g3YMV>49cb+C5
z?1!L3li=9b^*~H2e#LQoa?1JYXdfNP)C+NH{<^Epu74CdqL!BeZ|Y7&ev~>y=Q>!n
zg#(h1@?FqFV~To>=u2_m6ugZ4ZT^L)L}AA34aYG}Wy4i6;+TdEb=mjX!s*U>jL_?*
z^h#>Y0$tWB;4fx>Ensst3~yeiy*exk7f%L*&>2$}Atx6$RkNm9V`DP$I-XWmvZ=H(
z&k^138<K>q55HiP>g9^Osg<g^^^&?#oA8`myMA;ZqKHhs{dU39peW3ypfU5r2ewfv
zGLE*vZMHL7eOKzN8}$}GwRg4#B;n%w2cX1)QQejq4?{2|kydR@rj}NfbcUGW<v?TW
zRY<B8^TeR~zR<}JP)z4}Kqtm=hfYF0%Syhx%1*wyBf-h#mPA~KI~fZ`xvEySo$qw&
zBFVg&ihMWKO0kwt$g9eNGM53#K0b*U$Ig7I<V)%0MHf~J@U?zXSR^jOT{11*nNLzv
z@<{xI*P?AJbvlOlXn#nhZU@TE28?UCb_<duOn<E&w3QiwepSBGl99+NZinj~Pg8=`
z1G7S)4RGgvCY)O4rf^3s!%GEU=flB{!iR%gJq?F>>KzVkw?^}a6}Lf-;5lfszOKWd
z4ARggleMYAzzWyq4arC2UtASMVfhC1aatp^7_;P=Mi>Z0w<g#nMxLlb9xU%ta#4{z
z`_f{_toDgQY7VL>65Hp_zS$MUomw3NPWg@vq%&_gI?u7}tG#Us^#_D&SQ`s<)reDQ
zrAHLz$l!)Grd+S+x9b|aT5x22l;gV6-L^hQ&i3`u5wX`HJ>De=r@nTGH>7DjIG*J^
zGfe~|k})^!$MmBepF1x^Bq6z*3%bl-s)75!4R^e@>$_0$4vlh92HDp&nc3Ry5-+q~
zC}=g<Ec*LE2BBt}hQEtr!Y#e5!6vg@WfzEEJa?KR`~T;JD?;qA41at0!O(|8KRdKK
z_#X%V+riTVKNI~2(Nog@AWexs+xL9$KkZ%Y`QGk7?#e{|z3^+0q51FKF|uu8Hv6uv
zBHI9B;)%_3zUW2AU{UJKW}zWf-kEB>KwzJzGX<|9XLp_-m4pks#1_-Fg_n~W<*kW!
z{p=;~u#2f=+|F_u9@^1+Jwldq%;5nO{76T^+44!S_ICm@!6ZWDb|x+|MNvM@Yc4O9
z-y@_moYeuhD|QdOED2NZuLc!D0ekI*4m+Uj!iQbRFp$Yn2n?|aZ}ibs0}Y;R2rlT>
zc0&+3&uLs=yrWma*JyD3Lko2dEzHIO*7Q;doOk9zD=;sIsX1GXN3u{kYrl|?1l7I>
zebO{LLo>`*sFvme_o8~Sr9zM#-bqy`hpQnZny84U`J;KZ)I3*d?BofUTz6&yr6Hv}
z+(6U1`t$4vyP8fdCet_ME6NQynawPxmY^w%$`YJ;G5zm}#~rmHiJMb!+wk^rNw8z&
zs9;rt-M!m^^11=d@XkGJvb702|2xp}vH|N;^NwEA<f;i8VAV~nv@L6QtGcN@>{#LK
zTc3yheF3U+mWBQ~X^tE{-M#}JZ@2loVqQwl3i0gjvoA@)`S-J=Kgbr@&&S^&`_ZJ|
zp(`3Uh^~e^8(3oT$QNgnkT1}M<XDt^<s5{PU?;b8_DRk*NeDH#<f}8f!>ODkz|=qR
z4ED>9pKNs3>=R_Vkk>H}J|_t`>}W7ff@wA=!bQq%YMFVqF_}AP{ZP__v`e^JzI}`w
z3Psur;OH;2r{-4g+DWqd4pr_1^QxYCFtUfXyQ=Zfwo?sq78#-HX#`h*cVBF|#_`K=
zt@%Uir#8Q-^=>$~_T6`7m{>m`28OP2{7Qq<3+@)gPuJn}>cQeIM%zxkw3SVZUUo{R
z6CS%1^5#HhB4rFW)z0RnG{<}drkj=-*pe1&>$0ksd|SiJzMhzjS!r>!&`2pbUMFVC
zJFyESqfKLyal7B{Jt_*d`_TKL(+0n9$&rKqgRvscayxYX;HL^5LinLfUXBHt$-N*7
z*<}!eIguc%{2+f|enLkGyhOYT-aH}-D_7c8;F*^;#XDAnb4ro>?8BrF3`m>e>4@M|
zaPzPztS+{b-cdoPSJ+dVFdcW`aEibD8Cc9S2m3yCy29T3>3-ySNw~T9nIyPW%QfcS
zplmW|ZRV1hd(RPrzQ?)bAp-1sHnAh|Jpf>|!M|3`1>o7pCIB>oJEp(-tR%e2-!%Ij
z8%!t<AwZ37lK^fww|;#7zxf~oLy!OxKmter2_OL^fCP{L5<mh-00}&41eoXlpR}f7
zdyoJUKmter2_OL^fCP{L5<mh-00|%gN&xr&s0v5`2_OL^fCP{L5<mh-00|%gB!C2-
zd;+-tfAaenTZjaZ01`j~NB{{S0VIF~kN^@u0!RS&|EL2<00|%gB!C2v01`j~NB{{S
z0VIF~o_qr2{eO|zKNsLX{6hjr00|%gB!C2v01`j~NB{{S0VIF~9$NxmhztqR(IMwg
zn3`Ht^@_Y~ZoJa$I%1X+M6UlwVt+2cfB1(4kN^@u0!RP}AOR$R1dsp{Kmter2|PIj
z2D^_&{Zj(>`v0#Bv0s038i*}H0!RP}AOR$R1dsp{Kmter2_OL^@FWwMjEF)s8lRd<
z7R!2tbFrTj<X->(wh;UFlS~HNh6IoR5<mh-00|%gB!C2v01`j~NZ`{;pu0O7b@%@t
z3b7AAy&SORNB{{S0VIF~kN^@u0!RP}AOR$R1fF05Jw4HAl-&P+SoqsQEI<4Q!#_Hl
z8Tvm4?+<)?;LZNO8vWhq8`8fMe^fl&XZKpYFZBFhJ*(Zn)qS(;54!ZoZ%57w-+*i%
z!{3$HL}BG*bZ9DDEvl9#7uDM4nxUG-oTYBambzBb<ie(E)--EuOkP^f$Z2IIn@TJ5
z&9^(8y?j{|QoO?UTaf+SB}pjQ6ChWzSd{fjQM)Uz8zv*3PJ)@=AnsgRNoEu|wKT6>
zm-DIl>$y;Cm|uD^FE202`A~dbKEVV)lEisV8|{ONBxLRBCef;;`i5S~HOSRYCd1r-
z<T*cMgG@voH%WdpA0*;RWzCDi@&hVU!6?<s71_@c0;QHL&7}PiG&e6N=jY|Q<%R4b
zVDWS2<&1JYBQIqa7Gw^KcjqKw*&b>_u~E^iRJecIaUjV71gH6447bg-S0y23N5YNf
zRK1rLdliB5WR%NFx@D6eBqiZZTMSoPFf|xE#oXF<2(>&WZ%yu{_8@=$;R1Y#BS765
zyqyaW<bSvTuU!;{YexZq+cJL$y#n?_l`TUqYel_YwzRvJyfv|^EGTmsd7L+tGG7#h
z^Q%x&(KM=5CH>-ZTNCr?<*RZg`RW3U=}WS5J++!yWykE+M4qFRJN*|V;r#9eD7Ro#
zw*wOa(h%{sCR0nRN;*SGFEhoF;b!SpNz3<-l3+`9zR(d!!e}=+!IYTu9aNV@Z)s*t
zH!9Be8W@9oLZ+?fm8%vCM!n+beJ5qhm(t6NjP!tu2g8nkff&+L1csxe)Rtyf6viA<
zLqvpjbe0a*>{4na>)X?oSvcd?H(M?TUgtkJF9|nnm?Fc1g{`(*4l5JaxYO#wCY>>#
zT=TIre@+sn-kS|1@O2M#gd*^@kIZ~c8VFVN=ZWF+tj5)JYB8C<Azx8$$jNMGIkg1k
zEGkQ2LLomfqpYI%+1N;j7&W_09#=D)!yP?s4)%z6g0VGtYAnygY?(<dDk0%K#pc9v
zG1g~Ap-={hX|1A`wZ>sgMx4(Tl$7-4L~<bmCe~#OBaN;&^02g6N`g3{Vc9~l@1GTg
z8mZQ?9qs}ltf8<VU{oCDS(x|APW6smp^CJJJlt`0YDN-n+1Ee-6&5J<ihjGU(Xr(>
zh%BMND%RZf6qR5PV~Z<!!zd$V6Ace413etglQy<kw=ABKgyfE67F|maWQDTae%q7Q
z)2chs{Gdv9)Ue9wG@wDBxq4^6Ff9qHeX)HH1Xeaj2G$<nH{@u*PLKyF0C*b%cbY6u
zo~QD;8tS+Im7owR6W6c>soKlICA4#8N)l$@OM(Q{_WRZEqqiSl8^^-==<S@{Hz^4>
z?SnvbQQt7hUP&#L<g}(1<>g9gTYg>F?!XFw7(DtldzDyNnb_ib4Hn(2N=7yf*ic=N
zkGn~Z%hz60(uyA%l%^DQ%P@~~);E_XMBy^C$?}We=VmQAE(#aI<xu0j=l$M;Q!VFk
zPb#P7WK%@!ezU4KCUe!?3ninCkRzJ|u8P}7UXg?idnQovU?%Vi$&oK$d}%?fAk7P$
z7MYsGatj3m&{jUY|Nmn+!oVt!01`j~NB{{S0VIF~kN^@u0!RP}d>08g_y5C#|B=|Y
zgxI&>1OJc!5<mh-00|%gB!C2v01`j~NB{{SfhUPTPq!55f(G@t*Z==Qi2X(EKR-zg
z#I_&-B!C2v01`j~NB{{S0VIF~kN^@u0)j9Q>FtRQMY_oTUwFzP@;rP#=X}EtB!C2v
z01`j~NB{{S0VIF~kN^@u0!ZMAA}};?O*q)~Kp6h^@P99*`~R~4rv?x8-5&UdeNyjF
z_Ffns9NO*v?SZ!jzrVZE^+#O~BEK&Dp0GUlUP}Z6*Lwame)(v`zTmv(ApGu{;B^?z
zt4{oPeSGGldg)0mF2*l_FmXqi@Qa%QVSG8BZ+`oOAD0W#KjcLxJ1;*g#*ZG2ym!ob
z-GyI*|K&MaP4nA@zQcDlL0A4<{P^+6&e3qq<aIcH?c~R2TS)Nnzc*+7kQkpn60t?U
zhy3?j(AQRYzvIoF<3)Kd&S-uIhX3jdyXP4(J_i*|`+Di;CSQWLZ+O9nXnl|L4IG7o
zVtfju=x%PVD}`74Gi_SyrfhC34NvTBJSD~t9*n#{?Wh3+!+!_DT@E21#cOlO`?b6`
zGBU3QgBMVlTCFzWMbWp6(csC3;DT;#Hw4uzc$-Z_*gJYPS1^i=!OlDD%*F!N^ioN$
zY~-}Es+SrH>ilhE8$+PIU|$05wX|~8RYxLnz2^UpHFu~Vmvhs)J?e*&SIf|dM1HU@
zK5;biz*ieTm4m?l6>V3G3yZu|)K5?9aDo#X1Ff6lDwdyVxLqY~T251FL}K@?sAC>K
z7;v=HFV+7t7}iujyhDBU1meAa$$K9n_ceDRRbiBw)p4rG<dsQ{O2ud<liBNO_obyw
zgkIUw-4}EQ*31`~R^v#Nt7#Q0c%L8pX0jkY2QPVKUXSOAsg(^&%aLK%D6R9pHRd&e
zUa|7Law(ZzfY)P^*8=))-t#4+Ent!*-b?q0@k56q-zz#DOiVEOzc1XaB&fq4ZQE-l
z1730KwT|JtkKU}Rwb~uS1RDOUWl7^}@HWwgAXBXrYYkxqqg+)hjrOOV7V0(V5^&hu
zu2Z4S`ns;MJw!Xz+SCG@OT#+0Xwb|}tyImem(&enu3Psrq%5g53$#(I<Wv9?ys8m|
zfLLf%QLmA=GqXZB>C31L2|*XO^a5z6`^HiE)#c>{CAk!6RpRx&zIftL<i0<9J2Sh}
zspOkTm8*HO4$`?ciRxY6)rZ9Ey}WWAB_b(%+m}m~jhIxgTsJ5&cuydMVEI5_{H24D
zx-T}-17bkn-=|xa?IrT3cw(j<vy&+H!vX?koL~Xr+ede9^M4M};+ePh`ElMf*fbYV
zDn$_{gF_K}z@Kj%j85C!yySOJa}ABj!pvPZqaJ(ML>wT7;(Pj@?{hX3w)M0aKXN3p
zd)`~PIJS`BAO#<!_0e^+7u=y0A=z8K;?ZeFt4P+^GX|#PF_*CWVq*L<;12doFin`=
zJP>+CD2aHgrQ)>A=Jo@LCaUB$Z#4XF&V+q<*s<I@&pZ4{a`L|v=1xoL<mH-P#Z7jF
z*w<i(!X=-$+SM07c`|atmF;#3rb{p!@KV!CL92y-c84Txh&(|?(kSFi>_j3W*)Q!T
zoifYLOB3P!$T?8d(2uTAdHGOWhw2xUL0a3i8~~Ylq4ff^6=s}9yG;0LrfK-|I-{7D
zUe=(`%2n1zd+`bK{(pM?-x-7Wfdr5M5<mh-00|%gB!C2v01`j~NB{{uc?8J*|Dcc+
zy8e9VmxtaM{L{hWz`q`NqyL{oe@OZR=>_rU#Y;Uu(KFutlU=3A*CAPIa3=Cr3kuyg
zPaclg5jZp4d2mXMA3hv;KkFQ<s8v08OWUUZjE4_XXlRGWDD6)1*b6n@c?X=6fFzuw
zl}CN5M-F7}R&}`S$(+`>C*yFG$)0wHPa)kT&e1xYmK9qat()4GaVvDP4jGyAa{K8q
zF}{2(Vqf%+aETwLT+OJPaM12z%aJpBZC$OGEGBY{_78J#%)~Zlb<Y-=B;<&Z_RFYq
z8NjU~pRjvI#ds2gn()uVnQUIkb-l=ho}kh+pX@V<n5^xkfwDo2@#mdmzfaI@61I2}
zZeIX$!R-s2<`acyf=Bm*&K)eS$4OsLLOtoENEk1R@mUZg)<&-w84`O@rgnXncz+}z
z#*ZC~e4l&0m?@k2dXiS--%`Uip_7y9c6O!h^+#*F>R$?Gj7|$%1Q)!V^W3KM-@3^=
z|7~$)k{qjhCJoGhdzcQESFQP?&|nDC3purJZ8BDg5i^K)u0Avr%EY#iBrqua^53q(
z={o7RhLStJcmihjj&5z{pnxJ=7uVGiYj0`G0=o#wYg(ahYB`=H5DVAAi*WJ01XF~6
z#_pPAVmiX-^g*$S>TxlC9xlrrWJZPOr52d18>OOV`WHBk^9Dsza&J`lzUNL5W9k-_
zPZU4jcp=a)KIeGi^M4*|R~vQO=Z<@(CO14`fAJ+TuFXd5TbzbfO=C+Z{v^qbHGXyE
zRw~yJc`ahoeZHMAY{KRO8+hc{)WlG+5`fnu*Ie0SpA+M6LIYO!(}18Twvv}Rl6_xd
zCw7h<1N%A~*@ZDeE}eUx2ScsphklNV=i5LY{|=W~L5-nnbJVJ|FmB*pRWmDU$ywku
zTR7JtEHloUMk_I7g-xw+D_5(Ry}<`{ChU<H#dsFR+fAoe!*`j|wT(Mah&G0YgG@2;
z78D*aei02l8JZmZ+K7ik$3<)6UD>_)G|^f{XQ@Lg9of5Fjwxyyvi_3GUF1ZMit*I(
zh<%Eq_;3qoXnY0T3o+i^xS{o*N2W1n0~2}!X?QFqZ1Dw<1-fgBBMW8iJ9PGCX?r(r
zW3C1{LVe>Golm@dM2ydXAi<3deswwOVA&7wZqwy*@VzlNb%Pg^*f`vHN1b0ZU9bAT
zkG8Wcv|n4hj(zXkXPi6gzGp6MGyLLTXki&TGzM+gKaa6<Sg<Coyhsy6<K6-X9<mve
z<WBuwmo_d50>ujM6oZliTM%SuvTcwqn9bX#+f?BOu>!0I{S^tXG-iF`N)5$V>uV*w
z&@Q8M{XZgnzX1RJaqPnuJNu6&IQD@MiO20<_<VF~DiV=~q~F)No^gW&FG!NIQgq$e
z)=cN)-iTVM7;tATtU78V=kAS^^qMtt?#+86YjwS3r79!mCPqid0@K~-k|bnBK5}jZ
z9Ad7k6?J5E1W3W}u`&6w8zhg7jocp{0q4Ll<BpuO%(^x@vZ)vS?|Q+g%o!!agly)<
zntEb<RGyrefRE|vQF$T(Ws)fnVv~V%@>lBP<CACPWJxov8YH1|_@i><{!LOVc+q6*
zUDs;`wRA&M%~iPd##%Hg*5=5$@lhgwbu(iaB}=b50i+d^qd=f?3sAgylYHIMmW(?i
z=hk5v3~hk>5VaDCyEjrWN+q=l2MT`VhN%~?8V*FL0H{H}$<w1Fcb&qw;S;1Ib)01i
zf=Zm(4Vt<N&Og-+EohhtHQyh-$EOeLsImsPD%J@Jw*%D4Gp8rinW?d})3Xy}r%#_*
z8=EbjnH`%sduCQ!TR%N@c6OHZ3*7ctD$W^|b$x?$Cq1xm1cz3UPPsQyhP!P+rm9-f
zEK3_J!1)PmbV%+3+=FeDNgDO8UYmnGf(BcKH#F1mNjMRtq(!$B;ujvIB*6>nnpT2>
z>CdYrwE%ZpRzT%OU=w9P@S18SOZrB|4TXLxn(CcZYnx~i=^FxQ9d15fuBzm|b|CHc
zWl@J>XoVHSNk&xe5^y5Imb4A65_Fd3^*im^E&UxTKIx<i?EkE#dWrNkO+ac+k+xQi
zk~3PUG^z>mkjQI#kti1Qd)3e@)~fyv2s}MHLbOkthf!tfYjrZrh%{7nNj@k5s;q4q
zcdn9rP@_kp2I>|T0`590z{bJpwhhy$gWirLp?=z1PP3sfYZFe%HeriYqul)dG}hNg
z?n8>QTCGAkP#IBkxK-BC(I6kA0_s}LDAg^39d5p_k}Qt)JA-$_gu7KCW=k#Ap}7@6
zMl_bH(FiC;5P5do8K2-coAkB;C4nFY%zZF!G)EHn9Ueq5QeY5Gz@YR8(F7zSg9tbq
z)xc`e-i4~wQh^jULQeda45t@Ipp9_rY_+6rdm8Cj2<R$C|JDqPD7M=d3$#FDyP!s2
zU3`)?zy~S`sST5~$7N5Fs^*49x)CsO?dEqda*1v8M-_R@0P2TBQDR16L4SgHC_dqs
zb$9^6F(x^qnk&{JoZN)xJjn6p=W;ocE(f{>$EIrOW+EZ4ZOd-HQQ4@D$}n6_l^9_d
z2$o6)r#A>3Dwhnitdch0yXRDP|Gw<}E7MBB-R`}6MiqkZ-E)e%e;>+%Tf4!y(aFHn
z)>F`lpuG2?Z#)aQT!nOWXgD|7!9Cib8E6C12EurvI{2x!fy5M&DG%%nU9F%Ie7F&;
zm5e)|EK9fw63b`~aY^`TS`v1)Ae^PPaOyEw5W=?p+wS}SKNMmgqW`~-=9mLZMFL0w
z2_OL^fCP{L5<mh-00|%gB=FrN5bf!Xy6^vw#Qu{I`%key`EE*sbt3^JfCP{L5<mh-
z00|%gB!C2v01`j~oe_Xn8(t!}&dc;(Z7Ag#5;V{omEsZSCde-G{C`}yEyV5&e`EOV
zVPVJ~{Knu94qh7gzW%M~&r5$@{5|ohzR&djcyFTT=Xzf1{;}@QcKuCA-ud5q$8(Z!
z;e&c~Xv%%$LS~-hm7ROxLyxPZF3IF&#LD&5YG#$ZCGtCcw3mI6f8!Gs<cYJkkGGK9
zJ-sK<=u;RZPm3quK5dWpcKjG`;{4?J<kZ;s?AZ8Od3@&F_{6#Cvk%^qgrxl<C`z(e
zgx3ULb)IH{n|Do68TvpPyptL9+tC{44WfBDIX^GYEiYsj;nlo*dJsnD2w)30C1Ke<
z<VjE|YInoclQ$^1kNdQHg3C|Dc}gva<1(<jzaj~%b{s?~YVdG_26rvPTUlKxoPc5g
zqc*QBC>ce*lwMwR(#h9eQ_@P10G@_)22*NDKCudE=Q1*cILUpY2_A@nHzi@kP6SfG
zbAWph!Y`FP82G59vEGn`%XV*@UUBZwZr3C3Bbv>R)Vbl@o~YlDglYSETC4w9stgYg
z)XVUcD^CwYPwj@4(l5ZXCh*V}PY3T_7lj-533$_}GCj#Yn4Gh~>A>wf@DRxMe0uq+
z>^=buhBGjl-t(I+vgLW5Ub`j<*&WAF8`bR=b<0~5vSr9ylc}XuC7mJ9s4X)iv4KFt
zQ`)}iBwyIag~@O7st%cIX+BUS2!2>LSEHFc`)T!;UyllQ)Gh;ZIzKrNt;%8VF`*J!
z;hbmR;gN!^$#(kJK6`_iXwT4l(w%2P?Xy`)cwlEcY)j|O*jEd<!)@nCMiLfxR|3%k
z$}GJAw6FEy<pNLhJACjpkl<r|3ZBcyKEDcjdfc7>Kxue%v8>TqFc30?3C1WthA|$b
zzbBFl86{0kNQ)tqm($8pa#4|64|S$go^9DVkOmvRd+}kt+kPeZ3F+FvGlI8b<oD?8
ziX_~zbHIeGci_P@c(=Li8lTXb$9vn6Gg;B)2h8l0)kpqb1D>I|bX62Cp9IOX&RZP<
zRg)bd<&sv}ur}e5TQWA&%1SnsR^~Z#PHtHguH0}`x(v_Sg(1*q{>c;O&NN3K%;&!S
z!FddwQ-K{@l7wqE5w!_&Fg|Ea@SN)2p*qiBH@yvjKD<AgC(+z)zrRQX5(9RSk}fJ_
z=Q+h1=s*B8dzGkD@Ec~xj0{;WR84q9ZvDjZm&Rv{$4B*If-bj6aj8{#3Dy_9p4(gy
zg}EBV;jU#!aiKc-i<|Iz2J$>)C1ZD8k%TMuk#Jq~;r4ykw}*$~dCMug_scM?_?rsa
zV7+$9c|*^>r3hB<Z640&%F&l1TMKV<L*?+_9I$Ue{G9C`_U8%W-v9qQLhN^9U+;WV
zk2#S55<mh-00|%gB!C2v01`j~NB{{Sfk#PTHL^agO^#2WT^n1QoShsyJv~zxJ6kNS
z!^_jwit96lGvnh^(}|vFc25br-2MMIh1fS^-*}XAVp&K42_OL^fCP{L5<mh-00|%g
zB!C2vz{f}6SmP4_Pm_RXH+%oT@YKgA57vYPkN^@u0!RP}AOR$R1dsp{Kmter34Htn
z$n}5R|9|}4u|_0-1dsp{Kmter2_OL^fCP{L5<mhUCjs35f1I?jE+l{ikN^@u0!RP}
zAOR$R1dsp{Kmy+#0{yXX3z68jgxI&>1OJc!5<mh-00|%gB!C2v01`j~NB{{SfhUPT
zPq!55f(G@t@BjO{fN%eQlDZe$f&`EN5<mh-00|%gB!C2v01`j~pArHmzyUy1ojh}T
zLY<i!J3Bo)F?Ra&nKf_{aAtOF=IohSZEgMZ)Y;itcmMxw0d500`6*$8%|rr700|%g
zB!C2v01`j~NB{{S0VMFLBGBC(jSfMJKP$`&vAe_na=1A3CqtEivHri;-y8kA(dVSU
zC9R5oELwek+4nttpXvRj-j{pc?|!4}54z0A??<*HBf`%?M))h=6NNXm=+IPJtEgp7
zw$!zfCaYCFcT3xrEyMeo+nSKa<gJP1LPkl;ndGYrikzp(@^V^PN-ipLX4wye^!ds0
z$*HmN*|G7n^7zcT@riTOXUp%1Lg6M<S2T^PtXGQKT{*j=<Y-AbwNNnX6)UF~bB;Kq
z%K7y2RXMdZuUwb&ZPN0p%WjLp6{U4`q>b&WfOuZ9g}b6~V>MiCYW_N>6-}k{lkqCC
z58RQ2WqSsyQHw>&igJ3Tfzw=CNoEu|yOdhVDtloiFE9CBm6uQCgYxpDc|ty)IM3<6
zTU(MaV|Q~!0#*+dgYayunuKvH+pbH3VjtjE7RXEMr25eUZ)2T7c7D}6CoD;r-hBXd
z7femHm=Qt72d&c`1*uE&(sD*tuBTQrt1Q(1yk5*hXJnMiFl4T#Q;W&;4f%?4Lr!Kh
z%c&(OWl>qmIJuXy3k&_DklWY)ASWr^i+8$4&M4Qz*&MD-p{7yN^393e7J+E<pfi~i
zj9a#bdbJ4c<;HS*$sVam!iJrs-BhpWx9b|!RwzlW8FjOu<qUI!3~IfWD{AX%y<|1_
z8%dpOmE2VuCJE76w@}d7pjdu?+RvDhaKoMgA(Jqc9wKC2FH-562NWZL*Ct5bUOJM1
z%kJ83Q5f3*p|TEi84}7{YN@Veuadrznabo<B}3vbR875Nt)Dpl()eug_^4jw7F?=I
zLc%`iH5ZC47^Ql-qLwVpq&7a!l%AK9^YilD@<MhI?1IyvkT%Kl%B5s>AtR5=9IH?_
zBw@}T2-@*>&8Xx^@$wzr+LSBxQmNLWNkP&J^6`83kIUCyQ__kY1nV^?Kd0JSMG`JM
z#%NLBFv%b-85<jVWkW93$jntTZt3u~Ma9;3Rg<AklFF+!2y@Cg&Z*QaOTtAP=3YmU
zz%~?fMy0gvbk)8f`&pTR4n}~-S1*aeTs^?6S9D8POY)Xx)^wvn+B(sqm0pqy@&qLt
zgp6~Vm%1eiizkDs7#*Q%Q`l6^nr4lSIaVXBtYlMZWu8;i-M5KW5c5rjTum#~P0i`o
z{j+j2=X|B`j9Lp`c2p--d!yY|eMc2@VEc+%h0v?*Z7dK4W^*yUMa^>iv60&Zpw|Gx
zJX!D<r6R@1fF(<-W0TG*$DdKW*(98dyG%H;LF_MCCeW!WSXHsPsR8y@Rg?R%aPF)_
zmJCdZAp_ic$zb=cOTtY@4?^;hnVKn$8dI`f@<d~%^yY-^EH2k1zV}59l;V4i1e};c
zX{Y#A`~ASq>`Z!SdndCyov-{g0Xy%~aoMb2WLZEAlc)3#Uk$(_LaWs{>FZ{R6IxYk
zwL69hlbCDs=~AR#)69k-LSRj;`J&L25TqA!YTer8OCQANIE6B?EhJ%PcP=`fT_DsR
z?Ly~%Q50?wD;(M>F-k$~F!PhUV`Ab%yCx>Q#m=c-fa=LctzqZH<_uPkt#?mt((}yA
ztu;|l-wapVICO(*x!WCnjl9b2&#982*_WX@vgK)7rqR{mo_9a|njJP=`w=vx?Jmgf
z%}c_f{UR09wYN-NjC}1^xlDb|4uvVBVN=F6X=MK&iM=ksfB1(4kN^@u0!RP}AOR$R
z1dsp{Kmter3G9o2eLV845RJU}VtjUJXh<3wik_L8k`74Oo~mh-HEUC=*RJ~CAnfk;
zre#&nojMi7pPL<@9ltk1!WSWjQ5-opa&>t%GjhM-pnMXpi8lmIojZN@-u=Db|Nmz~
z?9cYqKunATkN^@u0!RP}AOR$R1dsp{Kmter2|UgOhI^tfMuW2l_xb-1h1iFWv(P9T
z5<mh-00|%gB!C2v01`j~NB{{S0VL2Nfu5eI`~1J~REH&CHY9)qkN^@u0!RP}AOR$R
z1dsp{KmthM6G4Dn|EK%^3HV0-$3G;11dsp{Kmter2_OL^fCP{L5<mh-;0Yr@-v5W!
z|DUi<#pWOZB!C2v01`j~NB{{S0VIF~kN^^BAVBv2&k1Y7@b3)2IrMLbmIl`b2L^t5
zAl?64{a=j!QglZAhWNGKpY9#(d9(X}>AuzdT-WW$zl;1JWZ3iH&I$V<czoNv53UZV
z_m<mr1N=I<ZUVO^sDA^lzZ56sqw<u(I3)o;VvL^=8eysChGu#GN*Et0CB3XexaWqA
zb-D6r{ez_Kr@=$VQ6M$9sa7^L@C8!aTr*U&nA0i+bGr&&)u@Lk;wfZNnNMXGUtL&!
zbyZnV=D@WU&*7fkZy%6^Rr@6<mbg(0O7cs0eJXMNdwKQODr&X1X#fV+&6i(7AYWc4
z0y*C1c#f#-WA;-~!S43Op{|v*yWmy~oQr{Pw~}UQIq);%Izwp`0Q_<h|9K6rml{)r
z#pD$3bu;umPt2=!44hhqyjV3>K^1t<&a4*fVNuAD&UIxE*m-P8gUeb`ua_;57`zy@
z?bajqkR+7tiC|p0UPN5iDvi=QZgW8GiF=i4o=cHd89Xm1jq$|02yw+jh7#Ach&^Tx
zN<z)P(kLKtz}eh*uj9cFAuA)#eO{v|OxAYNQq5}VEYg0?9w5GBPVG_PP9$}_EV^E>
z8e2htwv=u64ZB|wu6>R8NTgP@8KGxVr@<FkJFD89vqKZRFISQb{$_`g?c7><TE;cN
z-q7ZoXD^nR6K}*bhTR9jiPP>@byEWuSX>vEkJj$5+fh+i%|YKfo|)X4(HAb~*@xWL
zY5ADz%hsPQ121n)`qc1JVB$M>XWo`1VdcHUdr_m)kD(5E^Wk^7!|Xm=B;G*dz>&<U
z;9ktswv1b$>Dl*y+%#YLPLJ8SG$@17Wimarb)5N*-A7#fdfp^UYRw|1zL--jGM(&G
z1~xmZ`7LHi`zgCu5(<tRD{$xt9vaOpy#StWT^F?Sx?yViMnxu;5`?MfC0SpW$El-9
zPd4I~l;&idH-lqM;<1*xoaDxHmpk|E9!a?3J1%V_QfLXm?ULqN8@h%w#42{TC@5sL
z>nyfhE7Cz%`1Z$NcDCpQ#qNR%nWc)y)ion~UY}&_h$viK2EiNrOw#hesT|+IWDw7F
z$hK(<u>Rk1*8fIz+gC|&SqVC^HJMskRni$^+ymF&WG<vr0`bD<%roQ*nIFl&j$2~Z
zKOgz-xg93o$)W`O-8!pc)*~{B>!>vOQhIq2oGZIt&09FwvkyFwgq(dmu&#k4O)ux3
zooI40xT_l)!7ys)Z~M+yiG3jcw&}W|StK$%8K;@eOD<0og1^xZHVNad|F-W-Le`En
zR?NEBc3H4>GY&hwO1-5)gS&AW%&mIo_`8yDe)mpLH!aLtA+=o|1$-a1gU)@emp&HP
z+e6a-Dm*RxrZ9MC@Hy$#zGUw|>3ybWxBIVj{i*PqU8z`o;Qt-`X8+XC{~7(~(WQYO
zojnH0e|+{>#Ev;yMEbI7YU`Q_n@X)l9k4PXuQu;Am{^+LRVgN%(d-0Ncq(}~A~^Nc
zZkKe}8-klq!}*=7fs;~jU!0quU%t4rH7mvwiOBoM9qU&u)xn!HB(p4V>Ky)cKD4~a
zha)>&5PQ1H+H7KSXyN44A><UB#Ay|>K48xym=$VOuT)yawenfo#?>*Yuj8d_&<@|x
ztnckLVUL^><Ex2?oppMK?D4`xJy+*Gig5H7Xm2zdpK!XSVyO0nI+{_SQHFiOj-3_b
zSK<*n<qPSYFnNX2okq?*338%lTgufYrz?|pc;(GxqyBz*G*8c(5<9|-7>~yzyHF=+
zfs@7gI2+bZudGL6kzgS~6rp)vMI%1mgD`vej2KT%MC?;er?fx5Xx<dfP*BXw(VWOT
z{ct;`Ii4ttha+LXFfGQHL7+fOnsYE`4$YBgS}IL*TBe|O(lNW|gdIIC#*-l6n6Jj1
za$#a_4)u<Hnul3)+MGY#H^&iR&dk62)hRJPF%kK|HMq?wn?qg+t9EmO4vp`=W><!1
zvq^3;<BUc2)Z~dcjUdxc^TDe(Nr5?l1VR7!C&$bgZ{p6R7(ahFV!!0;VtC=^xJsr$
z-ibo9_HpvKqaj#~1!mOg=)}Od@FORo;z(1)pqMn<RLg<rRp{$DQ_-#W2VQlo^7qyq
ztE`qZ({euM!>a3qG1_V-fd#CN4*HgiS=D$phzXjb#WA*viS;tLAee}dwKBdW#y{2-
znf=Ze#rW~#k=-e}#8S6QI-EMd%B);1F+aW(h82oqpk<WHN?N{}PAw+WH{>hI4LO<3
zET@(r_oA|tk=e^FK^ZO9OD(+IZ>goa?{c0Q%$9DIc;Vp6o_MbJmvLT}#NgM3e;Pk>
zB(ifc)Ob&>i~J^c*xD8bO|QVE5@?Y(Og-UXthO{<iLo#A#rLs}zBwVrKcc}!_KR<g
z_r)g;M;`d&#2b!o1L&_)&BG&{u9b;Mz}3m4@xzBB?`Pcs?C#oW=J7c@dqH%1@%|^P
z7s;ZnX%7*c4<>d$cTtQVI~Ms`%xzb&HDjztvO!0KL>;PWOQj>?!Vy~dVk@Jq{x+g%
zoI%b_aFoYfCL+z|CA0Lh_Ks2E4PtP!SLTI(CmSG3Q_JD@xRc}^O}mw=!NS?Ue(Z+x
z_Gk%PIxiCKv^Sh?qI*~R&xLUJ&`_sqoVm)`a<SHuwU0&BQoYHU0qvY6!qH6=zy$OB
z|DLbtV?rc=1dsp{Kmter2_OL^fCP{L5<mh-;FCoF_y3=){z7Sy01`j~NB{{S0VIF~
zkN^@u0!RP}bV304|D907JV*cuAOR$R1dsp{Kmter2_OL^fCN5S1aSZV$?7kZ76~8$
zB!C2v01`j~NB{{S0VIF~kU%E{aR1*4HOzwqkN^@u0!RP}AOR$R1dsp{KmthMlSP2|
z{}Z12WXX)uA^{|T1dsp{Kmter2_OL^fCP{L5<mj`BY@Zc_s0uUBLO6U1dsp{Kmter
z2_OL^fCP{L68Iz%AlLt&>$)h!R$>D~KRo!a2fsY<;lM)wpZEVa(SI!ci1cOg+v499
z7kYoK_kM3z&u&j|_xE?7>-uMq0soMIAiQgzl!VLnq3F<5vRITEFQc;K+Eu3hKbsx7
zGD-L5=lJ~O_~g{s`0UvDS$TZs-1x+~>9gDR%aWkj2U=F|x+ZO3yXU@ic}dPQQh~=!
z&z)|9SNF1=5C!chRIQf4A1~YXSA4IZvSr9+t*F<_7Wmngw<cDV1!XQHkMm00vQJ3D
zqTNkfQ!q;Pa)n>6TczVYv_a_P{JcE3ypUal*7#z0o`S(Asm#H6&ps{*S-Uq}ap2Mv
z`jQo6Z<PnxA5#0f_UA>Ra1ScaR>6O%CQ~1)G6mrHZmu=Rki9ye%qXl>@~V>YvR)XM
zuf3+E74Ns4T9uce@jU8x?Uy8Bxv}w{44t%`>m!!OJ!OAR6c$egEoanEc00GQshTy-
z8XI%mV5gOpY$~nHbNX=SfqhI8=H4d?={mR#!EsC5^8BG^lCLf(a_W*(s&YNGnptho
zqZ9pk*5xkiaJIF3Zf__x`I&vvJOOhpLe?G;g^MfD0Y%fOa`Dfnm#@k`d>(jC*JkY(
zA5!jQN9A6zWl>mR)sOYwzF!5rWFHlU%be2oThPvD>=$5C*(D%7O{Rj$#K$}>Z)-B-
zD7jIW*`{>p*V=RH9`^UlxK;kBoz3&imHnoDL=rCS+~m-qgM3e5I%M)FUDETKm$yF)
zI+=yW&66=?l!~;wgU;|(poP+hdY&_!Ubhc}qF!lTg<E6$(({~x?UVLrBw>1o%%fCs
zsrPQT#CH1n5xe>3p<b|`7lqZ(fUR%nl^jermR{MQ-1j^I+5Pr&lAzfKq4i`f=q+DL
zPNCG*J6R7^-4I$$Exeit#pPvoHAD0HPU9U%=ww#sO|AB{9hZbV_O+ld+y!ZNMahLF
z$hotY7pfQKF<7Y4&dM&OR<bZtF3l^~<@_Ep=VfOd+0YJuDLI;_mBZx9okVuF>}Sb5
zJ`@NZ7Wd)06i(maKyRAL0dzl1<AM0H+X0L{)GF#)Nh@;v%#)0IrP3;nw@@XO)wDw0
z)N-w&-9ZNRk+onZwWv7D8KM#-bKO}@a$~vV=j=nGkYvWg?Yd#9`yK`-?Pnyxu!()4
zMj%i;r$kTR=+-D_spf`ewO6_}8C`{Q5-?kq(L60aKbm(d;Hw+n|Bu^vB!C2v01`j~
zNB{{S0VIF~kN^@u0#7~xy#N2n?_+Es5<mh-00|%gB!C2v01`j~NB{{S0lfZ?I)DU_
z01`j~NB{{S0VIF~kN^@u0!ZM=CxF-gpZq??79s&8fCP{L5<mh-00|%gB!C2v0209K
z|EL2<00|%gB!C2v01`j~NB{{S0VIF~o_qqh|9|rP7+Z)0kN^@u0!RP}AOR$R1dsp{
zKmter_y4E^NB{{S0VIF~kN^@u0!RP}AOR$R1fF~X`27Eq-^bWOB!C2v01`j~NB{{S
z0VIF~kN^@u0=WN29Y6v|00|%gB!C2v01`j~NB{{S0VMF`6TtibpZq??79s&8fCP{L
z5<mh-00|%gB!C2v01_bk|3kuW2t%pCe>r$-;2Q&9?*AM8hoe6r{ZHa=h%fj3TJLVp
zhdu9g|3&xpuAk{z?>Y!cALqZFFWFy`gzS5<=+IQb)Kp88D~6@7>jl-)jY{rqqi$Bz
zl5DAKB~9L%m`f|kj3Q@}uP!KZ>XN*)oRO95snyJ?oL5VlY2|p?^YV%Qyk5-9sill^
zSxL)R)2YQ|`i6W(xgjUBndQ_H6uhV`Wt;++vI`6SqjKIbH*#z)QkfU$R@N%k6+$yx
zX0K@#E2mn4L~Pl$dZD1zYMjW#`N{Fgsj>0dvGKF=_{_QSiF4Cuui6(y;mwmkFk3AW
zg3TZ(zrv<!)--Eu%xPv?S;?l-$~>pYIs1#EkZde+zopnG><f~h+N&U%S}e+XrKsKI
zN#}x=KQC9*vPQc&yOdhVLLV>9E7#?Gc16iGH!J66%#mz)d3lMa1JL(gM~>#5gj|UB
z3-%WzVZ~ksS(6}Z!6?<s6}hO_i00)?t!9+Av|JhbMcvTUlBJmx2S@Mna&mrNo?BkX
zE`m%g%4qIhd0x4c%r0c)ahcO6cAtG-5-!+NVF^K#iXdK@z%}<JS12bV4QPLRNi6%E
zBuv}!upsNEcDt@s3fub<!q3$p01rTqJu3+__H?*jy_PF#>uSAZ?W^J-8&fkK;O(2g
z^QwIoROLO=ermItRkK5j^$}PqcW`8s>tW+&ZC7E;GzEisu=L6X7@wvnx2<4B=fMPL
zQj1E+Sk@ie!;R$*xVP*XQBZCI!=h<aDK3|zS%(p4sT;YiiTU*M)xg?^Ej^Pg&r_Ed
z?K6^~*edBJqq<EEkCzK36WKE4t;y8Vs*=u-sbSf(A6^<_JBS=$=E;2EN(Vm@@~X0+
z%w=S5tb8fGyy&I&ih{Oqg)D$N3wGAAkyM#nGZ|PszX-2>hmBs8$7Euu>J=TvUC6k4
zIbLbZ@%rRLtgDxk8e6?c>td~|mxj>sD#(v|D7c+%uiB?2A+?i)zL;~Tc{it5mP{vj
zPrf!2yqBfJq2f#IF9z7Dm-cpZ9KTup_LL;tu_Mr|v_=LRZAW9{j0WWcZGhg}nn*5W
zU^U`4ma+_*msXaNFvyzKvV|_)vnNI2_APHj?L&>|IBS|~Jz1Kxfj8X7><LM@<tXw#
z)$+TflP}uipp!N+L2Y$%4~tF@l%tc6Qp2|GQ^bZH02v!rP;ybNZLS%rSp@S$3DOQs
z&0nWI#-?G0Fd4a_D6V!F>{mo#rp~FlX2CMERh>>`LuGClx0bp+Mp_u<RO2rT!Pd2^
zjjjO6Lfl{fk><JKV3vrv=B=9FpRz|`HSv81p<-%yosRP?(ZhSaLj$y>nKjtjG_N|y
z6cg%|W-A0Ks%k|mwcJ<GW^;44SP0N1EoZ`~J=wGnXtPG(E$5hB)23P}!Y+;9_L>H0
zbZeU@iC%)0rN->wSfj1o)e3cIyQrC_0i`!dtZk{K9PEu$FOKf;yl_|v1&bOQSJCd|
z90DX`C;`(THv#OcDjOPa!v-DZDi?QQ1hM;FHsSsMkMnT`%7z4x01`j~NB{{S0VIF~
zkN^@u0!ZM|5EvTxAz?UD5W4?z;AH>r^xum9kI|<i1@VXbexmy;vCQyq4z3RTs_^fH
z#K6E0Mb5<&iO9~FbF{8j;F`31O&!j-!@u67hil$BbSP<ur(8@8<ZRBn@Xj9Fv_5Np
zH!a4KFGlQR&Kb6IoEgYOPyERBtDurB=X{Wxg&q_7aiOz9=bBhzcPk~vCnh2v#9eYt
zw-vYtyzGqzUv41Sl%+$89USL<jMw$x_-b!gQ6VAkU=R-4AZAaOp&?xdm(d|tooJtX
z9_Uq9sGC}@@eC6p6Sll4#+N4|_M)SA!4){RtG#;;px|abSfP5bpPLsP3H`I{(ETeR
z<s|76?;m|tjK|}V?>iFcrX$8YM*}-lIy|7p1M1*Lr#1)FWLT5qcHg)=gQ&b9#_wN%
zd8geVa?j1FDR;*aTx(`-n`$0g?qU4$en!|sYhf-Lg$ABmS7`WoLc%2_M$CL-=c`x5
z_=O9R_hZ3`<doH_J><r-;EpzW4LD?PoJMpwl-tc8euIMkV=ip4ZR94PVl`cAX=^>G
zGPS((&6UZyn(FqQm!Wm33H#`4VtgeLu~!-t&buhvc=ej81ePER4^gOArLd~wr6CIA
zV-4<zuuoqWou$|@Mn$|lVQH@L)WJ*Ip%VBKar?3LDoj_z_-s64$AVRuv#w#UQ95O(
zsL-8XeqNpLi=T`~Y7I3xI^b0CTB|Ck8v64m=>j(_u{I~hr$BzOp;T&h==^=DRmpxb
zrl(2ibWFKdmSIspd;VC&KIqp`HMMokBv&hIxq8jH1(XSzvz3e(H(+fmt%&gy{5$36
z3|6z=8pG?kAST0+)?dpuW^23F^~%mC-h1b&7=Q7_$oF1!)zdF8sCJcB+q6UIP$l<i
zmtbBY7sn~i@PyOmsyWTZxnfRZ;3i{j!!WnQm(yuvi-okMTP5C&bCz}&Zs%<>hh@z&
z^@11SNn~nU-W1c6!@utAWn!!{dkLEf)~ZHflflA7nc4=qk2<sm11W+#9DysC0RMhJ
zh=UOSnl(GMhw}K#f|e;9p>l9{YO`QLNeMf$EXL=+dNX@H$I5s{&$r&L!g|4J&OqSj
zV}@_e%T2~{FaYetk{G{sA!1(%#$wYBj@}F4<#PXRo~RdF$<>_HZy76LbF#3kc>Vv;
zoTFeVNB{{S0VIF~kN^@u0!RP}AOR$R1RiGsxc`5gdk1Ah0!RP}AOR$R1dsp{Kmter
z2_OL^@Ms9&`~M#eM=S*iAOR$R1dsp{Kmter2_OL^fCP}h<4gee|BrL;plnD02_OL^
zfCP{L5<mh-00|%gB!C1S4FSCW|Iu*7Qjh==Kmter2_OL^fCP{L5<mh-00}(K1aSZV
zIQI_9h6IoR5<mh-00|%gB!C2v01`j~NZ`>B!2ACn4M!{m2_OL^fCP{L5<mh-00|%g
zB!C2vz~fASx&FWOILnE$Aps<S1dsp{Kmter2_OL^fCP{L5<mi-6Cl_BpA+s2v1f*V
zYUruKBLhD_aIXKSqW>lOgVML8e<=QiSP-MV{XJjp{)_I-uAl81jr^O)s_-8n!=v~!
z?dzhjdM7$Gl`IzJf>ElME3&0-)MU$$)k?*%R7*EV&{|1TOO|H3U*&udnU|CE^YYyC
zLUwUU&Xde}Iip<9$V=IU1$lCOa%yaRc5M8tJU(-7eB#{n*|+R#qA;sNrSqmyl?$q+
zZ5ZZuPA|%OrKsJVPcL7UQ%m#8bvd7!zn%-K${FScB+D^b@|+6ne*1MvxM?2_x69Nv
zpe>}DP8;W!kO%oP%4H?(HgaCMl*}$<<Z)heU$e8KaOEV(lC2h@txa6z!lr80G;3^3
zURut`X=NpwN-OichIQE)Nw{bypfD9W%)!N$cUs<>noBFmjM5&URy7Knd3hN+ETEg0
zPvl*|`O&<aB5|H4#iYFoQtU)qNinrYDT-=sbInlABI(nrTG2}6%fn@3^MR;d&WA`n
zY^MpkWGksB_b4^1eC-Ib>G&K4dxc2!QY(oj_9&6HUDeulrjvj#gJfSNGEBFUVSJA=
zs3py`p!?d&;V0tCvU|^7mW1R7&x0%lQ{xVmt!b*>ndGYrpyZdFA){PRt!7pmbh$rI
zr18e-)pTkxnZ6-kQEte|Y-TyN1O+WBOJJs8%n?)7KPr3b@8@FUeBpywnyX&ZOsDBy
zWnOV1MQzIF>upmEt2#}QZg0C5mJ!|3trBQKbGUZb;slp9%hU^A1Z_yw)V6Xpu=%`#
zD%zcvabVX}uPjC;I+T4ZUKF?cuh~nYpe#e@JJW(uDzcW%nStc3)6V4JcRr`%c?RpC
zy(kHqGtkI9S1~GM^@@JGu6Y*JQC_oN(sD3=<-DHeSW<pMs`Q~qVokk*^4td76ZV26
zY}<>_<Thf#B&@INpb}*AaysXs0=CJ*irSp);X*6+6;W6@O3a0`9B`WW3d~-XUU8QV
zWv!^!%ND5Js<NQW!K}!gL=M_tmV~l>gf^b8J!)E^ZfZHdJiQ{j5LsEn3lWDp8y#Md
zG?eIRX^XYVrk#Sx`W6V4*2>0~CWrKdE<Y-2Sz{L|*%gI$2`h%TSRtvotCW}R*Cau=
ziN=%e57pG%Xs@`sUZi@NT}rKF{fWN45Rj`?&xfXcl8&#eG5a!+mKbNcj)An^l5tBd
z)wLRoCQff<ug;V8NOOziRV5=sAs4EqUa{6s9Diwiws?G0FDB$`uPJGT#L1~u$2jv;
z=$?H^60&x0Sc6C=t{nT)8Ytx<nzCjqu!_A+D_=KYWeiJZqKv%NG6@gsCFBTZr&dd4
z+py=Of_-4;6x744_?mRyH4L(=pz`M_Z9wsPuy(|LI5X6K6vwqRRO8*wgT)zBSSx+<
zCJAOS>1Br%8e0HY(RL5obFg@Se+*jetpZv#mre>3v|awRK&EYfYG{!e)c#{wA$T($
ztS#7<2a`IJpw+~?rI|Ib{fxFSnd;7-qL@=by_)7(sMSU9Ol@wDdKp@m%I%i7?N`At
z8qg_4vT1XRb(=(|rnil{Sy4+ZCg|o~py|1$aTAv5f*sunZ*+Lo?NQ~~?PN<)JQM4w
z>z=gTYX-Y{@(S3CzBAYVAH|Ly%RmB100|%gB!C2v01`j~NB{{S0VIF~9zz1S|9=cS
z1w}#vNB{{S0VIF~kN^@u0!RP}AOR%shza2P{~j?=EDi}E0VIF~kN^@u0!RP}AOR$R
z1dzaENC2<@KZc!xA|U}JfCP{L5<mh-00|%gB!C2v01|k_1aSZVh=F2pNB{{S0VIF~
zkN^@u0!RP}AOR$R1Rg^Ixc`3)I|W5T0!RP}AOR$R1dsp{Kmter2_OL^@Q4ZE{{ImJ
z#o~|v5<mh-00|%gB!C2v01`j~NB{{uh6I@X|Fy?ZJQN8DAOR$R1dsp{Kmter2_OL^
zfCP{L5_kjzh6XaiGm&2u2L3Ygi~WD6|8Vq2q+gXjFaBKL#hzd3InjNqYdiKcv5Uk1
zYUpncy*zk-;I9gA?6uN?Or#J$emwHt14~^iX+^cRxn`(lF{c(vp&zBWw35syawhre
zf+FWbae4Vfe_k)<<<wF}xvZq+tLfBYGJQk7qTG;^+01fk334wgOBs1-8UD{MEcB1c
zdBfbuvANi|p!`-T>NU*_XQ#zlnp);YR5i1#*J`>^2_*IEl;@R8$?QT#K20Pujgpp^
zGs^Wq*n+94mIiH{2MU?gq7rIKy;^J)o3LlEiSa}tvNPr&Rx1_5QZ0}oXQ><EUvGvX
zha)?r%oNfJCCCv5=5<oSHVR_=(4oj~(vh8@CI1)0a+079rK_lApR`x&LsX;Vo`7=e
z1cBeL5TKP+y~G41>~m{kTs;-BZ|jw!b~n4C<N{sX(oCX0Oo(30(ZHru1qvy<lv>Fu
zFgE6u>vDeY1?J`DC3c|EdH^&t`k+0dc`qhmKd*}M6%hWSBYbN9IwLAi_$Hy(sL)i}
zhnKKdDHA()^J4tesmS|B8!SyLerj0BnndkT<AQQp4Tq{~Y8A`t*JfMcSO~3V8Kzdu
zt(ykuzGwJ6b3&4My~2%buf#Mqx3<kMx~$b|>IQG%fEiLfb@s+nV&`^Fj2}H3d4GbA
z;+h3|0Y(TK;e5%3QTyQ?A}(u|sTU?ZIjCKBLnj+Tx71Qy%dP2Pfhrc$TAIm?E*f>1
zobobEK@0x$;YO{O3>ETmVz}j5dRfb9Rim)U)a8MKPIKakEzgl}Ev&th$oZ&QYPrfw
zmAL(u7(f4F#6IYcM8*)SCC#+J<OV@+HIKU>GUugt2U`$9Y=Iw_u#esp<JX|hRbHL$
zZ0oi?r&WsXx1g-+Z6)C6=E*~6_0haTnbZUp$|at#U;By}zx`rlXP}LQ>;P>o?zO#$
z+peg#va=aI@f(`b`-T|546skNX$oBy1dwm;MQVRBLF|8!mvpud*GrSIdv82k4r)(=
zvR>bd9KMZeBZrqJVZU-+jK7(P*qXzGOmjhXWTnIYgjv&FbA*$8g-X>tV}{dli)!po
zQiC%-g7ua&S<=N$0#?W~ds_M0(UKTX9j85Qz76C07(1?=^=4TsufclM`K}k4pj*wP
znOHA_D?AfHB&UVt6Yr+QcoJ0YSQ}NdwyRn|Dcf9Y5SgQcnPh3C7E&@NLBf7n7vnde
z$wALDg8GHItL_|KmNO~dZrQ6$q87z)x%h*?12mfVHUkM`Q;g4otT7)|b0tiO-pq-K
zr9!n6BC#{GA;ym#i@Z1OuaI-DGB5>g(rW1DiHQN;k9zCml-_D><ALg6QR0~#{tigH
zYiVXhg|)Q57J?}pW>oI33!)O6>tcNBP{cmu8w^Kss+_P=CqFmZD^HL(=%kPelLdd`
zmL|rhp{D1#HIXd{scL;sRk<4$Vi8H6P+d+`B3<l@pF9+)g%!{hlw{Fbs2tU5nNxIv
z`2Tx8vLVD?AO35@LqiV+|IWZq_y3pvzfMj9q65-x@$2Fj`u?}RmwJDx_kM4*XT96*
z`WIanBL4`o;NRW}T(WPA!s^lJ&{V2ICbpU;!)#HL3r4A4uE>@lm$b@;wJDz-pFOjx
zEGTmsdHnq3_~g{s`0UvDS$TZs-1x+~>9cm!u1bPxAA+Ki#iH!!g1l~+vbS23jfxyt
zWT*z#x6O(Xq}byUkSk2DZHU5y+aOC?D;rxH6^D);x=bTWVzR=K$%dv$wE6V%RT$XL
z4wmc*8Nv9)<atzP?207Zx3f)D!ew}C(|t4I7voxnK;qDt<t$RU3{|@v73}!VP%GKl
z6=UP7wbO>QE3<8@*o^$;E4PtH?GkCE9c`g1Y_YkjTG2|dhjAO*soJzDlBYtq?OUR7
z>t+l2oX+f25_gNjQQM^bwkTXoK%I+Z@R-^L903ty049E;1OdjKf>G4uvc6$D=X#tn
zR9hE?#WzVA>Mc$7wNAD+HMs_pf~?lu(Ck%mz{n23RVBk@xZr;0)L}nsZ%V>>&hYZ9
zYinV`qsOxcZS~A$#%}czJjHs)-jIZ8`xrEzSVLbNYFO10EFD}E7~bzB=jY|Q<%R4b
z%(i|eW`RNX?f%*AxXi;6x7Q`%fqf8apED|ODr3poUA<;O?=hHbuHMwNf>9~fNcWcY
zQc0)3+w_WjK|bLK_wvN}_;^CTv#FW1hvhHGv;H4Xv}bKi5_0wg1rtmYbgc!Jnh|ji
zvjpu!f2r1H=jNs1_890=kuMqD@O3Jgw_lXoNam&COV(o-Bq3|ZDKn20IgTN{7T9?g
z*PfKiV-Z@rK18j0>@`*nkIQ!?hu89b$nlP?ih@oi_pEab!d+o77;5mnDAzR0J@*Dg
zTDfX%^EKKz8owYPKQ3Q;O-X|pq6by-7v*t|0>5SFML{9vxM&(xI)sBBa5ni{lg{G7
z-TI_1$;$QAYG#!k=rkG6-rFbboFpiA9jYi8)orF-B$vE3N&0+iDz&t#q%#EDa%0J4
zx@Sa!{Qx<f^-cj~x{Pa`M81?>UTiEhw>8NX_m2G*%%HlxLF9F=7}3Sn9?wJMF_~D%
zdPToo_tt5Q@F4Xo%uoLqMSgv%<>rM+Pa4hq^s_hG_=g>x@NB$&-M%Rashum(@;SHV
zOjV?>oHZ`=6_wOi2y$l5R|@F5PBzC(y{)#vwEKf`<{bg^ZBD=a6-l^bN1$zKjcoI{
zsvb5!RI8y!w<ePd8MrD+cmIrdAO<<QUP{8{#AeNHrDVPKo1$=!3<+n{xwFRJ6^sr@
z=6Y0EBEE`nM!)?9`wdB`JIdaP)_&Jj_Ve})P+3R*wkkWhPnBhOJ{koi`+wo7y>DbO
zB@#daNB{{S0VIF~kN^@u0!RP}AOR%s$s@o#|NqGo9wkNsNB{{S0VIF~kN^@u0!RP}
zAOR$R1olJ#_y2psgo%&<5<mh-00|%gB!C2v01`j~NB{|Zq6y&s{}bJPC_EBC0!RP}
zAOR$R1dsp{Kmter2_S(z5y0pF_k;-(Aps<S1dsp{Kmter2_OL^fCP{L68J<DAp8Gk
zB4uIt?ZMv~oETW?|9_&HXha$me^NZ!_s{xX>;09UztuC+UFiD5uI<QwhJ^UnIRU$8
zr<c6Ee*j*vNnUG7-~J!?u^{jHB%U0opAYw4prQ8+GqHJ@eT#Ka2=f|iaE8Hsaka41
zBMCQcF)WAYXyZGO!Ykt8GHmX2i$bd0AOZ22=T+aL_2FhWdfDSO_uZW?Nx1B|Oi+E7
z2@J?robS5ldXRBOz$Dt6KOOG-YtQdQB%$WK43yL#dIc|A1AWCdmB#aT)AC)W56|kp
zeV0iWa!^3)fw%a3u3NaTsNLQX$a|6xKu_`Mxnff>TflSEw3CNh_s)p@KoTzO-h#^M
zOOJ`vvg-!NRgiXXPz^V|lQ;f)g8Qy#yw~}XH$9V)+Uhl|9@FL*0Ygr=EuFah<SLk8
ze^n9|gBO>x?|62Eq?!dTYuam5kS^$|kj^+xZ+J4)?EBz`-~sd#eOWl9^z57CPOnDZ
z>rRBBuLmbz;T72#84_Non&47z{lszWrSaL~@lm~)pl=B$rKDCJ$9%k6-?HBog;_mp
zL`WW5SCi=s8xKgy_zdKk4!hgFCke~;VXFPjW0xW$uWsHG+lPAj;kC11zat7W<h{({
zA^3`0dyBE>Msj<SJKF5q_O>LP+9fZ*XGa_BLWcWpZPJ$=zERu7;rdNwhL$HfZ`eKB
z!E@EVD+(LLGmon*f!h%3W%Cu{&yvhZWv!^!!Ab31OWvAj=c8v=vhTpV+9U6gcjwns
z@a*Axh44$4ol-$f{SeP9sB6`r3ii$OO{w}%^hbk0y*Sz*rSihJu$xf`Fj=3X@S7qG
zzJWhb_(7e2!Z#vaf1=-QB~#NYPECHruLO1OKa9FYfZFDklMBk~oN^*4ghmp%<gtka
zBszZFk&Akfc6z1Xd4ay4i8pd8_LeBDkd9adKLvhLnF>z&b%Hw97^jvgNJPw&m+%6l
zb3GRXAw0(g_E*5AfM6fDYec0Qr+dF_ueYHco?mP*Tg0D25I`&PAh$FxO*t6XyznM|
zT8wK4n$i;U9t46xWSsu;fZ9iFi>w`!w0Z73;)4=Ap@R^A(qYyHK`c!l5_zwpa;@0z
zGj<J3_WKt=Kx)OBYV*e!Wy-Z`O}RQWaX~q))(YYA%B-Ckix?Wh{3L!y@84=tgYc-_
z0`nAS_i5XdgxU9(gMRV70q<ocM;!`x*a_%3xSu;(dSN#!#(t0NY5!~N!`L@te-`@>
zu|JCaTQ{iltuE$70!RP}AOV)ZPbazugh*71Ohtx%^GH`$*!9f>`1qXq%X=e%lb71a
zxqBnvD$W`?_vXEkwYpxiQk9W&6Qd(!fnGD}W<euKh&$zxb0gp-a9ypaBcmg*vxVPd
zWAbG;NFEy-xj#Ar9)`dh=*T(CtZSnqn|jgzu7h*BIiqBlkj>m!Q%{VK%99fl@G(6-
zDo-S!OtMvj*rf0Ex+Z_6K0ZEqMoto6UNuNU<?vTLBlmBTT8p}~h4kuO*J}l}bVF0k
zRZBIkMWbSEj+`4GCGuA{Glo&J^r{m;S}{2a1S+=x#hW+D*DY<yxHEEYT`kq14UVr!
z5_gY}qzXPqHE@MkC*rwO)X6iaC)Am#v9r^&6Jw`OpIIB5EuNVjn>l-CR$E&?J#}_=
zmcUxqwNerM`mO65gmF<@SL-EcCVYp3TSDsINEzHS0+T9uGqWshtWZ*GHU0l*?`^=_
zNYeYxCP)eZDT*^YJIkG&omGq4SqUpr0{Bg7wX+ZjX&~`KBms%i>@3hgH_0C2ON|Ca
zYF4{xU}o+0CU;KGKKmZWzO&DcKk~%u&BgK8-Pu{^9LI^zj_t(u#ZK}#$*w;VJ7;@6
ze!Tmf@2&oBpg~d`&WuL-V>BSpRrS_ezk2Jfszz5;o~6#$s%p8&;s|sk;EYtFlL#g8
zwxpD8-Sq$_Y2J{+rk127yF4f4qyh;xzg{m0d8tS;=#o%V%XB#>DDi^4UNWu{JW&zu
zuBbb#dKoZ@;;hj*gNqd*FRMEiUr{D0W`3oz9*dd5Wx~XCElBHhQmd_@YSo2FdrQ7&
zhEWOSQkANGp;jmm6EgzKILV4yDHrq;D@|nth16kzH|g+Ys%*00N}2BYtjPCB-|?YA
zW)adXS-K+UYAPLr$+R)eiKQa}lx20JeD^v_M>$#`s$>FQC90~B-=s4bnTU0zTq9c=
zj8pz*W$DExVd@5rsW-|6v1($otFyeeHh7<66opEK<S_Gk*&Uw=HiZefDjnggGTK#D
zs<5cGc~It(RH6|52H4>F5}~?3<OGvFIDu19y{*s<CRWL1l0*uS@A%3Obp*&~ghFBR
z8NagF>@vpBM<?JiEHkBMz+46c4X;s!*Lak?2Kj=ozP8xW%Y_YJ@XbZ<=pj0Cesm13
z;J~hcJiH>Uu`Yr9FLUSl4f=8MhNM*W-e`1+d3d9H6$NFJV$d0jR@XoWpR)eW=yS{g
z8yottGh112!HzyBv8D3e;fW2FltuyU8*}n3PDRddN^C@7e!s`MW?c<cR`o7@t-K>C
z!_+^Pg>}~37-N~W2`Mj3j7$y-;?%_O*yv<*I2WCY4o~MMrl(?LGG7nV+h2z0?Q21L
z`_m)z_9tx8|6jZ4?Z0&P^nZg3_x}P%GynhA|A+nmssG>g|5pF6_5Vw{gck^a00@8p
z2!H?xfB*=900@8p2!OytMWDO0qa)CxzlZhrkp3Rj-;e0;-THf%{@$s-2dD^LJ^f#&
z`TsxS`v07M_y14*ztjI4H243f{r@{%!V3gI00ck)1V8`;KmY_l00ck)1VG>&BoOTC
zpy__N$M_ox8-If#<L{B6@wfYk@wcnn_}kfK{0(#->FNk~1laVyJB<DR-oc7Q)*t`^
zAOHd&00JNY0w4eaAOHd&@HP>^`G0Q{Xk-HdAOHd&00JNY0w4eaAOHd&00QqI0nGp3
zLHNiT1V8`;KmY_l00ck)1V8`;KmY{ZCIXoMzfGW#4G4e$2!H?xfB*=900@8p2!H?x
zyn_U={{J0>kE}re1V8`;KmY_l00ck)1V8`;K;Ug6fZzY$CeX+R1V8`;KmY_l00ck)
z1V8`;KmY{ZK?0cnzk~3RH3)zJ2!H?xfB*=900@8p2!H?xyiEi!|9_i6BO4F^0T2KI
z5C8!X009sH0T2KI5O@a(u=W3e{x5R<zeR6&fdB}A00@8p2!H?xfB*=900@8p2s{h~
zP6c|njt-9Nc;>{3zy9k$LX_3AG7^nV4o4@3N21QBk8~Ij!+OLUvUFGSjmPHy@8e|d
z*k}9xuy6QirS}he#hzd7NrWRICHQl}t4BWB{p;OQ*DrPco6dMZ3%tMMX^MQ1mv*}T
zY>2CB@o?X0UXcV<;^mSkZS%?5)l97_DH)-rZWwRHvRErf#%EdNhxx78%yJ^0O7O{r
z*~BWJ-7`&=UtHj`w%9CxI;)E}Bxi?=Z?c$;Tp((r!_n#C=oBBFJRgmnAD_}bTmL|a
zn{GrLU70AJiQrWsSCC9~Qt?Z3q|{}8VKK!gR+B5K6>qLRS;~X7P9?4+mig<;$@%#5
zE&f{K79UTi7LyAkXFjoz($g=b=jM8b%=GSLOt=;3E(M=Tta_8`p)6@jh=H6!W(i{|
zIiGN{^hK~%e<sLX*(3m>Qm*iY^158g2&yU-D{4k9Rx)?Upwyk&<;Cl~O~E42E_T+B
z^~dV(4{<Z~XBg{pWyhNp)jY43`K^)U!b)N}#YislS&w{Hw^U?{TuCTO`WLy95kygu
zs?}_SUrEd*W>WlK5&6r@i}OwLU8VKa|H=A5h?BGvj;%O$?NL;>a;6?L=#f@~_K$2>
zvL}p{nAxLVRNvauD)sk;xC@Q2qm)O2lfPYeZVUFrrQ8l`#kzv6w&<#v)<D}(twv*6
zKR=te98b@s_-K!RnFs1mhdHfN>!U(1OFC%1tu|Q{nUkk{uKLwIO?)mzP1WGiY-!7h
zg*Y{x78`WQ@UxN4`g?<%@Dk~zH>UmAM@(;7E_dhE*{*)S7S&yUD#Xp}7D+9@bhDoH
zXRY*atrfmfe@}>u*Y7r2A#eJTy<MqC>37Ny_vtaK^(U#xY5i1Khq@}PpWT#pGFw8S
zW_23rh2&D&?#j|ji44Wc*s+WTPnTFTr>oRi>8aavRcGr@gt+1QQx0QFx;I%v{y`cx
zQ>nGt8Y(tj_iv}8^?o)GxXGG>RFG&OAmj_YLOp(!Un?uTaZ%(&p}LVP3yN6fg_1}?
z1yNF}v*gKXAfP*HJ4S{{xcOpYIpL(r(8z)Lj^qM=+I0VP#hWg|#}{V3Nlskkhhsji
zs{a1^u@JZ37$&u=!j?p8t_ih*>QY`VxkNFIbobNa?z%i0_k>UPgu`WJom3s7Pfnom
zy&@?^xmuOWB^Iib3ljTlI!N}hRuO3&CH90PJ>eVixpZQME}kBa4beY7Hgw7u_MI9!
z<%|oxUm{G3m0)D(lrf(4#_^=4jC|N=!FpeaOKNPq<k-Gb9H$_y{>F(##4=7yqZxQY
zS%Yvz@VBlf>PJJ|iWY053d_VGo?$@u(#I9GjY38YYNSkDn>HG#_lCG@+H<?;q)I}O
z%wR8l%xG<Nu|oaH1nNC|X~DXMeQ04uYomn~>aWFgJ<M8v-egfL$#-hjRKQc)8RwfU
zX1sh_U8ob8)v)u;N!mtRcT}py-4RaLLkCgbi}it&w_xh?8An$tBx?=M1sVYl8S@?b
z<WK&|dN9PLcWoJNd3%|nJMMni`YKhwUq3=agx6WqFlU9fT-~$oIoq<Mvp(Q}Lj<!8
z+w(MQ$jRmuj(j$+Y95qYvkq!Bu0V4lXW5dI$_#KhF(=HN<k`dAF(zAgDrT>0CbkF2
zt>Y3lc{I9&eb4jQ^Z$YVuW|ifr#HMn00ck)1V8`;KmY_l00ck)1V8`;9vuSRogE#2
z9q8)p2>-P)|Nk?t|F8Q0?9nM8N&*5P00JNY0w4eaAOHd&00JNY0*^g`BV8RFP5-;;
z59a@my)^&;2!H?xfB*=900@8p2!H?xfWV_lz%&0h*8iX9j(zvBC;P;sKXLRz@6Yy*
z_55hhhr|CQ^!p+C$R8fL+5Jb|@9X;Jz)uBEc6^3=gFDZC3->oTPMg*`f&(W`1nL*`
zMH+b_zai1e8&z;Vaj%)HNX+H5PKdrz!}#hfD0Hun_pZTEg)Dz*Zt;?{YMs_PNZS>e
z)_1!WUbq&@o696<&ALx`gq8_&!GW1G0qvx|8rQp_fLh*^O2(!HvZ$}X(HGY1m$^pk
z!kDHgR!U=4Oj96Rwqq^oiD-kL4Gyf37E8JoY%OS0^h`yOwv3$+^z560xw@%M>!PBj
zn9YLPD%X@@HljWCnczT@l)TthNe53;uAH`S{Tvx>RdYp&H0JIH2hN-cd~RO1zNUPe
zzH)!VYllt22V6F!OYtoHaoL|s*QQmmWVy|ZTVog+u1h5{-`33>3Ni&`<cd$2lW(Rf
zNhKfSr~Fxnw`G`%i<Gj;R+M_fT&3EzRgqwG`BB^SWX#mIn&ea75v}*r!GS9$JG5!r
z$Y?F%7A;3Cl=GXJGA)@BR9Xp}smY=%;InPb%N5C++ORHHFl&(>`DF6TuLcKBp6sY^
z>c-`Y?E3f_uT8l^4rotaC0eVUXt2e={#9BKs%l2smhv@*$tO<am3zT~$>SYbuaCzw
zP1$Z2_sz1oHXyLW+PF8bV|!vJIIu{Wz2K9XZlc~jUTop7?v}An!X{tAlyJqOHIe!^
zwkERv>Fwaa@#7t@U-3E&JL_$4ecIO#SettKUA=?38BJ?NTROLf?7sM~Z0@0x&(p3a
z8NI)-JOs5dYMy2UMOveNb7PknMh~d&R3u*kde2MS060q!J@VP2Th@iT(njvz4GzRl
zb!f+JgYazbpc@t2v%_=sUaK{D!tGRs8F<1hdo*@Q(Qa%72Tq-`TC^vo=gW@Q3Or$N
z!V=!HtxyCtIB@Q%4z17jo1Wby=*wNRM6E*vPfP=$u`h&o-w7>P4Gug-dDQflLs978
zmY0)}?ycTWLhDq50|~OCQ??a3>vP${Z)at#=X$Nxjy&P6bOr`bD6=F-F!c_hJJeT^
z7U$Y%;o3F8BS_s#4OhZ8dNge}aIX>^c!4x|wv8s{5@e4S)xET^*DbR>v^+@}2`e;m
zue^KSrUUc{Q1;5(_L{zVTcHu{qov@$Eh@NRo4qF=-yHxvF)F4et*ycn%~w#Hx^WVc
z+jngxpnbIX0LuDdD<4Ez3)X{e9iW8^G}_<AxZS06K2zf>?Nzr<Y}PIMR<I{Zq@LZ>
zo%U;=)g6x$NV$2w6wIv6X{IOafKK}{*%QEc{{IcG{~P^pJgPT5P%aPv0T2KI5C8!X
z009sH0T2KI5O_=pbaiqafzD1YjQRg#Y5^!V2!H?xfB*=900@8p2!H?xfWYHGplSa9
zRj&W5{eS*A6dYv)0T2KI5C8!X009sH0T2KI5CDNkn84Alj%Pal+Bn`d&`V#!e{CEN
zYyAHI7wq}}zj%bpkD`G92!H?xfB*=900@8p2!H?xfWTu&fE@rBpy&U?VYdGt_rzm}
z07V7?5C8!X009sH0T2KI5C8!X0D;Gd0Q>zP^Z&=F!ckNZ009sH0T2KI5C8!X009sH
zfya^ne*b?gs~v>~0T2KI5C8!X009sH0T2KI5O~Z8VE+G@RXB<Z0w4eaAOHd&00JNY
z0w4eaAn;fc!1Mo)WwoQwAOHd&00JNY0w4eaAOHd&00NI00nGm&vkFIXK>!3m00ck)
z1V8`;KmY_l00bUO0_^$!_i_I-*Z<@F(PKY+EYkN6`aW{>M|yv}_k-d2(2oTFGFUkB
zpO1vQx4S;x`42l61ApAHM3LXVmv*)u3vr*-;^DqgK@@qpBud-#&I<)eQ8S{nCYNMY
zE|+-rPg&2%A}^FgUX|5?#OKR}TCr4}Sx&@L2|l?nn^@(u$=TIRbLxzdG@~cW@{0?6
zwmC|cKb<vVWrwmxyvT)-=*Z}Bbb2^C#YZR4M`P#5ry9@Hqap6v>(@wQManCZph~UU
zszR<HnR=z-m*z;j%lyJ(ichR2S5hm!+4p2+F-xULC9Wiv`RmKc`S|iJ{#xP|A5W(i
zlM5tnKCzI}#Vn-f=6Z&BOJh3~%fbpX3!6%;deh{~B~fRZ<y*o;K^1&1*D4esk#@5L
zI+dJHc$AP!s-$cQg-lgqva3!ZEOq$V#N~K;F2zqonbup9Ql&cGC7vFw#&~#CwB<y5
zd@JWu^>ab)%1cx=qEwLDR?O`3;&tAxmbO*mQ~TNa*$}67A7|!Tpi=XCMf25-GFfX)
zDUc<wtA=6nhINRU+zzkqR3u(4^Er8)aIGZf5;G}2>SthDs6HIxlG+(hldy9*YZ&P;
z!7=SCy^vf=+vcBMN@VQFmfh<lnbq4N9&=~GzC7y_^`Q{=!fQ{GE~cs563n7+#db4u
zJ2vHa*!JCO9?Z5Xg1XUUHSDre6|Gk14i{Aa@PbfP$-bph#uI}zLtk=QU;RvoyQ!Tr
zYq3_6@6;sIzvYGehLmA#?X2-}Q`%`;nNH-49ocYUP6(?_C(vJ^)%r(6+=corrz)NF
zMqTb!kxpE_CY{WT2I}@!R-<QF4S#ZJZt;@GV@TT-S&@7k#hz8+U$)MAgu#A_$`+?y
zM3J`2o03RX#hRSDA@Q{;wMwC;ZqS8IspM|ESr(RIVHrIv%g1MD`I*JJ^gJ~fN0iGh
zsMdKO=Rhs@bckEjo;T~<=**hLpVhzHjn~y_HAT#r5skWT3S<>-h5I8pTR#=zirR{w
z{WS?YTcJ|A2(&fDrgWL6bZs?lj_0rO@%l$XTwZ%&KN>sLwQpseByII{Me$X!ajpJ*
zh)aBKoK!U3Aw%Lati2mj6T<`gMO}(}xZrMdu3~t*Pu9Ac>!LM=q2anzB3J0?ef7p!
zkSQP|SA4>pe05LhYA~)*iN6me#gwwj#u#4z>Pl#NaF73Wg_~m(|Mb4qsXbl)aEM#c
zE_m82RVa;5GKF$}^K7{yDFPdQW_ImvofH{Q3d7J`0$H>5#_(6{PW@zvyV#)PyDU3%
z3EA7mT#*lGUtT3!&7kcFj@gLbS4!JbzQ)k`n1OGhcj|*QuGV^~%*=-`#+kQH&KeGz
z`Ek4D_<L`!-!2>eOcpJtZdh_V&)r!+5#nCf`pnD?&u!&Na}IMrM-vlU3)AuVI&iyr
zvN4yPVo#^NQ|Ck66|LWcK<_JQ0C_L~Mr!xKy$yo<_2aDX^5(5~i1ygkF31B^fG#1!
zM!%NBXp_IzS^rRoOKVSi3a?0OG)^#k?SnEfh3wA~I_l3cpI{9o3^Nq+d8t}`bEas>
z*q0J7);}2JZaR|;8~j<_#gaXeH58=MI<=74bac1b1b+X2`)+0+ClCMu5C8!X009sH
z0T2KI5C8!Xc&7-k-~R*sKhE|4KlB8^x4lzZAYTvw0T2KI5C8!X009sH0T2KI5O^#J
zoR|)E(bLk3r1C|1-FU``p6jV@^5WOe)lYVIb1!ssna?nn3nE`A(__QDpsG@_qE<gD
zetmfS!+~yYCft=TNJ6Pr;f+Ts>FId;A;+rtSHJN4?Dzjb|ChP`|Iq*C{vUiSS)<S(
z00JNY0w4eaAOHd&00JNY0wC}Z5qQ@Ben9BC&KEk`-U;YD-Z|5;%PoKa&i{Lesu-$(
z00@8p2!H?xfB*=900@8p2!Oy_L;&mm-y*OO3j!bj0w4eaAOHd&00JNY0w4ea4-o;(
z{~w~tfGQvW0w4eaAOHd&00JNY0w4eaAn+Cu!2JI$0t>Mq00JNY0w4eaAOHd&00JNY
z0wC}Z5zzPl@B8Q6v9Hh@ULXJhAOHd&00JNY0w4eaAOHdnKY{uO0!P*xbDdp6sazVa
z(sA^1xpZENjzp#Dsj0OoL0p>>Cq_qN(`!>>bX@&ZZbF(Ee|2zEE{TKZ2i>U#hX&;e
zU5ibRoQ+LPosC7KXJaE#x@;UYzb=(j3Z5_DlM4mm-1ylje|lcX%O$m3-MGLfOR7}h
z=>osF!mskNXeKtEnT+u9N~Iv(lycW(_1yUA<k`^){`9pMQ}c5}d_mrn_$yL=vmD`P
zHk5KvIv0ygoQ<;o_!VJIP-HU+DNzurYNjelCDLIeIx;#OogR)(rD9{}N2bolrq51}
zOiquCPK`|69=w0*Z*jC?exp9nL2xcN!4Yz?>D+j1ZESpcWO{rgpC8Mut&NS0Oy^>f
zuolhj4u_bGjgf-y8ZuL7CnrZIN5{va6E>N;vH$;fbN#>4|J@J2kwZfe009sH0T2KI
z5C8!X009sH0T6fu3B)?rJKQ&B_Pm`Fh<2`rJt_9O$AS6(BWO1$5(t0*2!H?xfB*=9
z00@8p2!O!5i@@73|Ht`%?=F*owjclkAOHd&00JNY0w4eaAOHd&a1a96|L-8wkOl-m
z00ck)1V8`;KmY_l00ck)1m0Z)@caL}tG=Ku2!H?xfB*=900@8p2!H?xfB*;_gg{@{
zA9Ke#B3#!`bWI)mgJbg1A3a*`{f?gB3I1uYc;rXB-{}501<vhHGKu=#;6ODNs6RuS
zN=n=5r9?(3NQ#<KY6U5?B`IuAO5;meS<lF#@wrAH%)r*@+;_|@C*r9DpI%5Vr4xK|
zVK%YKXAdAd%P%hQ+14Dg{OPQ&prKQC$oNKD(MMCSY~*vj+7rQnSS;{`7gQlvkXq%m
zUg5so)IAlyG?!?TcTbiwqGF{IR}#zo_2uMzeEAlCEpdyFr&Ei`1rjlzSV-{;i}Zhb
zZmwsD_o!<pZHd52z^4+cPFj98aXFrzOYu=#N=Yb6)rydpGOf|=Nb5?uR%w#O0xN<-
zdx2V&=aj;uid0q0iX>*%XkSnNC>CVjqU04xP$eqKELlM+IiK*5P-V3s`Dm6emqb}5
zbMpxk1y%65O#7Eg%9c=|y+v8-s=uzKa)zbPR8`WdNs5j4)IxcdBAsH4q-`mmkrZW*
z6miy2<kGsHX05EW=59-ml2oBoDw76OK+@BwW3mkjC<r;J&~#0=ZoLkv&28@@IYt8H
zYNjI4Mz1cTw5rq5Ao6LgGdOVZL_mAqZt8g<zabfLjE|erj&qIsT-$c+M9SC+jmGMP
zutw?xM#i+jk%22G0-T<E^-e)nrHoKh%ldayj9yPMM+Wyg`LXN4ffFYJ^$U6<cEX%b
z+-u%O>x7UFJuW2(yEd-%2L~j=U}T(}U3Ggr)5(cKbt6|66p=MCf?JXbR1c23;J(ew
z(e!N(BBj}>F;2E0^JGlU%*r|&(MGjn!GR1Ze#fpqN1=?VxW_kS^rq%S;{JBq3Uwmw
zB57C6Jvo!+Ml+9S7qq_MfJ|!N)z$VV<w<OGF}m?PYH|O=eu%e|^FW$AZI7sTXh(ws
zFOd4fyHF=1acTN@9)Pm$!M&ZF#~xUC<n=YJM{fgPc)F=0bLIN2X09zAm)-^*yu&h0
z($~Qsq6@E8sLi+SEB1Eqwu76{JIJ>Ek{wTvreoiwpY_<0<*l20^LAbAyRpCV`W;XA
zWV;~86pVh$X(xx)M`@E<n7nf&ps`KPrESMs$R+B&OEl(jjr%hBnxc<p^q$>l3syYt
zzi;O+tQhSjxxIxIo3%<;*5u$knUkkTX`$f2-AJJR)P7XgWsw=~T86**KxOb3xxd0#
zX<N(nF)esxU@_9J-}X|^#Ju0RFEl2#Bf)`4B=EUYF5hKQZHoPQKObhrWqxeXRRx2N
zJvuGZh(KHicN<1~M_f*7b10VCpzhzN)q88)n*kd)dcreRnas6y<i;ZE`{7G$;Ht{S
zHgUYdq|&Z~J6Gm3_O3c(XuYMnXDFw655pk`*pRpMk;r|mn+A->16suH`l=gcnj6-X
zf;CDY()Km(o6YKLo%h*sT<HwMv_o3Ez4}vHS8(9?@j!#xH_hMdsP+f$o4f{N2fPJ+
zUTc?OOs7;|OEG3ZncR-Q0hlwSjG+35+Xp6iW3aho1keBP@7@MtfdB}A00@8p2!H?x
zfB*=900@A<yP1G%{{P=mIX?exE(>%A0T2KI5C8!X009sH0T2KI5C8!X@F&35|3A%@
zxc=0!KR&kJ_m#fqkG|6TTfIBIp>S{L$dUi=$hRIj(e+)OKOFeifmp}4b_6Kun|NtY
z)Gvp)JK9s>zEOJCiXJ`ZMQP1^jDuIpyz7Y^UM}(8Cwk&@DSATQcqGvq!Y?Nl;`C(r
z%;H>ne!(p>wdj_4VI(>-IvkxIj!yB>$@9_J`SB^Or=AFM_wVZ}70X)^WlRr?xr7Yy
zp2uhSa<gKy%Zu0P`5}E_g3m){P7%G&XJ!4gd9FSi=CppTZ#R9IM0$ke5EU~e?n6Po
zJ`>_(?E>j1h$5da7iz_lmr#ZtFk-dott6p9&)4(%!#CDa2<K_6`0T7zdreP1?eVY@
z?+5XE{SrZZMw3k?jT&t+C{r1#Pp68SnHuG*?)mJTBcX|*Yt@NU+|yYne(O_mzGhmg
z$Ei%QIx`Fg{(;J5RxdA5#dwYR0Od1<9iW_M%KeqN5vYG6#Lc|^JQdh@#Hq=kjOz0|
zw3WQ9uMSD9CRb7`tyV_Q@4LABJ@TiAw?3Waec<<9@Wh|@DLr?ohwLJ@Jfr7lcZK>3
zL2hM}>OfS=6|)8iqFmCK*H~+RjJnVrq2ZvZCcL%Kn#RxnZPq^?;<RUvGb7jih2cY5
z6Ps<9t?{sY5P6)UB+9j-Ds8K*`zY?v%8-@BTw;b+Joy!_-c`Ryjp{=EoPVQgo2}gi
zxMGmi=<UmVYTVSy@*J&?F}5q2xq`$i+2ZVe0M`}6ep3B(wnZ$nX4gVCUk)Ahj}eFI
z`k6!E@Rnh4mI)q!!rl-XJM{}8?xoMYN(>aq<8fL^@7MRVraa)&q%GCzUW8#S&|(jf
zwnW^^n>}ue_#%ESpj1B}<ZkM2PAW*M^p-oiPu8b{+{{a4YrcN<LHwzoAJvZ4r$XFK
zEkYT@one}>vd<Y>JLc&#I&%WEynB4M_wqpBLD^(|l6*{9yJGs7T1mcBvqszY*v!e4
zjkAq)L%LgK?$tY3*@sIt7qaP1z*!jO<k9+^xSw%o7wZ$GdY`s<FxB-HP6t-rTpzWk
z`sNxrzXE)wK2B2{$uhR4p`(MNJ#zmkj+4^;D8Ao_yt6(={dxx*kn8roUlly)_N#<J
z!bgpKeU!B5Z9lp=lu?Ctx;_%(Ue%^eYuRsETP?h0PU#QaHN|iu?9a5>{QtdyZLWX&
z*thllO5cUvo}PU8%b`CAeM_h}__u-|IP#s{Ki?hi`i0Iv>AV&AXA}`H5I6(^bM@On
zF2fq)n!L^1YtF5qJS~7Xdti1)$Y_<j4dki2yA|SI)=pYY&RWE73AFD<IP!dYJQnEf
z>uk+hm)1CQLf`K0J@r)@O=`oYM*5vCdkwy?TKiJSD`yw2V!m3f)NfKxHlW?HK<W3o
z?A}+WWB0p5yk{k+-VIv|IGeh&7Eh(?X?DLJDfR+*_aL6d_Bbq05}LkP*=Bu5Z`<`7
zA#PFYHVb7%J~(Adf{QOLm+~X}*?Ky}-O|El)*ihNp1((owRo_N2HH|RMRoe5R^4Bn
zn#+-^`i>uZoq9Uiw#)8}iG{mfJqw$S{Q-8ZR8M@W)Llj_pRKP@Pu=}86-klwyRx3)
zBHK&E?I2tx>nV(T#L%>5jA@%~#|oADa*(^mCWEfqoW`9W=iVlxv9D(9y=By<CSR>D
zg}AhB#^$Y6*H#|~$k>&74?FsF{d$N?Xan97Gf(Vsp8G299vhhWHKb407lYhqn}=o$
z@Zk)~()9&uB~NKP57bJGeQ}0(!*lOrgx0=&=pp7Ze`heyu6jIQb0cvj^Ywk&bM^TU
zSJv9x!eYtW59=SGu{X`3TJ&b&{tBL~&xN?EHn|`0Z^|DTZP_&%Zc%a<kMr7a{aT2-
z@Xej1rsw95PEh~ZV((kcW`+mZ)5-8ht`5hwlH4qqXO9VejrfD_a)d214zMMYzg<63
zziR!Msdo|P{t?ESd8_YNqypQLg!xd<4=+A$)cJYHeYeF*)v^@d&xvaBdNRcAXhlz3
zV^(QS(3^kWvTtNtnVUA=GM=--4|$^tzq=J$w*Df!-BN$LeJS)Jk;^&U@>o*$_tTaO
zD-C9>wDi%cGGmW0f8*(@UkP!N7NjD&ehFtC3|WdL%ehf!;eVjtz?n4b2XOlA$J+J$
z{~^rkAOZwH00ck)1V8`;KmY_l00ck)1VG?nB!Kh(9>!XMP9OjRAOHd&00JNY0w4ea
zAOHd&aJU5U`~TtcgftKU0T2KI5C8!X009sH0T2KI5O^2~VE+Fw)(Uh20T2KI5C8!X
z009sH0T2KI5CDP0C4l+=;qrtu5C8!X009sH0T2KI5C8!X009tq7ztqh|1j1HbOHeo
z009sH0T2KI5C8!X009sHfx{(${r?V^C!~P@2!H?xfB*=900@8p2!H?xfWX5@fX)A(
z>Nv*redg%vy@lSc9yR=*!`aYJhX#T_cjO0;^mTu}>;LUac78q}cl<O(d&FPb>3WeJ
z={aK_JZYZ5T$M^<MnC?#nxWGf>9q9Hx^WtIzFb<9*K3NP%H@*hNKfAw-m~SsVp~ss
z-SvR^Y3)qCz>d|u;yEO<oo=opfQ>@6J^9yHar5+0<8*J|7_Kvn{SLgGu79w;8RC{2
z*GOICc-@SzoXnml?fNF8><|0UUA=~jLP=Pcl&b6OZ2jO}I*pZ1b<GHBYqha(^ORlx
za9=*!$Lg<yxSF<Zo1V+EG8;m5!#YvcK8?G*@%b31*YrF|b?H_C%~OYM+dEY%tHV2z
zI-C~@`I>%~?J1v=XQ#Cf*X0oRsupWDM;~3x1HJ7-+s%@-J9^futaVPZbA+}hd@noH
z-do=YaWC7)Wt&ahkm{Vb>XGSb*goa)vb7XUE~FAy63hHTGt#IpKd!z`C*CKF1JET!
zHIKLVC}LbQZ8GBxG7GY%l#9H6P&C=ed}214p4Y$0YkYC1dZ)0Gm`lv0_^8ieu+!T6
z>T4k`qt{_v6fJ`jr8S{eP!9!;DXX>B*aqiIr~Y(Z3UTv|xZ^#1>S1r+;gjfqo<o=9
zTN6z!$vmgq=_WE;<WH!(@Qb&bTk&-psk#{C%FJli<ZWKAW-3ChO2<uG)=*YVJJOF7
zHgh)G->#N$yq;%=%8y#DxmmF7gxqqghG3uO;*RBQEw;)JY7*LDJs09OwTq^TdKT+S
zxmL-@;<g3ASI-Ah&?|QrEn~jTx}z?HxR<nEQ%kd63PMgQ5U#RM?L3fnh9Jw7(sOgZ
zO7zyV?67I;JZ)16p+vr2AGsWcCN61vsdKcR32|xdl$XgG+3K4ILQ;0ABVXqt)jvg^
zx8Lx*s?@&4@S>m!F3T!PqFgJg(zfm!_jIr1#9pRz&9Bv&7h@;J%Oz2wMqBbR0;h}A
z?a$~_hKz&3OF~hqR)oBiVF!E*CDFL7+rFt?wOwE8Zit?v-1PvDWFuIYYK;2~=joQf
zI$M{1)+y*7G1f&xkgpTdYV}VNm{|SUHwVVTVYxZO&Mg+{h2&B?!J8-Br<W4OklrY!
znWPm7$#fjHGv?R_1xiU}x17Ci!C(7K{iP6h?enZ*8iV4treIcRDt>8>%5Yg9RU}rE
zE2$Me>sv7g84kPal0bRKFRUZeBd~%QLyXp;jUCBr1K#L%B-6O`0g5Gy1y%$_Dya^4
zUsa^4T2^Qb&PF_~O`eId*!vd6;7ui&ji(Z+<b1*&sIjD~tQMrEQnKqb<`QK#+4Biw
z-ofXxTq2*cB@{AMiKV8ggkNz=<qS)osgg%-&e$Fsomwc*QlwK%6KPw@XCy`0BSpMG
zV@bKReyYdM0;ll&|08}&1H}UY5C8!X009sH0T2KI5C8!X00AO^`9I761V8`;KmY_l
z00ck)1V8`;KmY_DeFE(H|Bhpy;OHM-AOHd&00JNY0w4eaAOHd&00JP;mO$fTN0|Fu
z<jJmZc>dL%@SxAKxWV&0g@gxbX_hLPU+9~Zmt=ZCpDqlpE5aJBvmG`U2M$X`K`smq
zQKB$~u@zmp9g6po6*03`E{K$Lh&S?MYYhfprs(%s8e4RgrD0iz2lW_x=A>(pF&6cS
z8B0TRouwl!JV+}bnBInpj&xmb-b!bMsyh`{>_Jzl!h>>k)>u_ZA*xc7^umS*tCGqd
z)@OO?ij(TJP)L!gRf%02{Ft;Y6e|Vk6S}ao`Ev2&gRFqz`|SO`&Vb(T_uTK<iiv;q
z3tyF@<0DhT<oM*&Saft$kS24n@%&hHYCI-RO^;2D_4QP_zCeuY6~nb7?+JaZyWF+Y
z`2&F;3dD|`34ZU<c+Yomf5zSBmf9LXb3pKWJN|=yt*6@Yo56wPnSd78*Ez8THw8JT
z2+B@IDo91Pz{6P9#FjBNg>XM)u6Jo&w9^#RCBs;y)D-So4?L7*D_0_<*vEqd@pzzd
zUteq9lz>%tNy@V|9_>T8zvo$xlWiaKfXhoQLTssx-*S;nV(K5_s88!4+e9&znYLN_
zVlIrKHs_N%qG}%q4osd5XwTTSYAipJi?mRlwnx~MXd?t#bCS_7aNpOqB3-eJWbRdF
zdSX_0y1;Bi+tHp04xBt0XiVyr=1T1P$bGN3T3kT~tOq0e*5ykTLEZ3K>r+z-K36TG
zY_|a4)ke~GMW#lsZx_%SA);N^-X9z&p9$2Tw4qm&@)nUU(JDaZV3@ZmE28J~rnF<W
zj;6@m|I)TnnquxQ)~K+iB&^b0%CO~shH9h^eKA+ZM>Itn2o9V%6KFi6S7?)zrmx)h
zcq_Ik_<)t$Zlyk3Ugc7{wf5cRYYlajYqo`mc3gX3aA1Bipe5{@^xe<GtT^8=?z`Jo
zpKp9qmQh>2F<6~h71QX|o(c|(js{-8;A&~)Rh$*<Mziu<;{GRZ1^GldU?u62-qwZo
zRQvnbrhsC#vmblCUX&tvFg@^A8xd_l`yh3rqXF%rZ81J@%{ul8;l8V_wfMyL*osfM
z+fwS|+SAk*Oa@*H>XzbbTD}*#zw5OY-$)0vmv)uH8ns>^)0Pf(OBa&8|J{5d+CA+(
zp#iNg7O20VS88&0bzh5-J&S5x>$8oDa<56|zSAUZE_Z&jIk@{~cRtJ>+^dGk+t&(G
z#-nJ{3UC@7ewa$%9S_tUlWrP7X-Vxv!2y|iys&NcO;JosHRl2}*3mP~uz^uiOzsET
zwt%LXyT~*EG{twdhM}x6DPe=Ip)4yu{n1Bh5$%@tWN@HLwcR{sa+BP3>A6n@e@ELw
zwMO1m_5)Te8$-L6wKvwZ=k$L3n>utS&_t-|EBF20hTjzY7BqRMR(!|NtTe4XxL$p&
zm)j4z*$~V}jwn_0id<>4Uc6WqrCo-LZN`e$VA6MZPJYPTda6xoYh$W*soO7^k2IFF
zXM+PXGl9>YYBw;*Y{+a;{*GaRO+B9x#Qm>c3)m&<AzOpfD>ruq-kp?u>anRLv>8`Z
zBwM!Mc3CVtuLPTN%gzT@?Y7%kY###?mR&)psu`hDDaew~9MIM_8ez}>cl3XQqknjT
z00@8p2!H?xfB*=900@8p2!H?xJc0x|JHug||Mv)1EQ$mIAOHd&00JNY0w4eaAOHd&
z00K<}u>W5Z5PSsz5C8!X009sH0T2KI5C8!X0D(u40Dk{}1S=Iq0s#;J0T2KI5C8!X
z009sH0T2LzCIW2!|6Z=n9s6&`jvf88qnV>m_S_2p`*0@oMrbkkUBN5epXe%eekgFV
z<1-y+xZk0OhyJ?R=m>I~k#OH=zECbne3AAw<3-wHAy*a@k=OT!sPbx=ua)FA+RnhZ
zLYpM!Y5NSmDAVSVY=4cB=*Z}Bbb2^C#YZR4M`P#5r{@}6kjoq=$wEPu6uxP*5z2}7
z-(Zr9k|@`Ts<f@HB<2z`DL#6^N8I)LXM@~JXGok}V%s!SDR*I;ckS`P7o^g<y1_(_
zM8~FvhxvuY6u+EUN+*{SvpyoV6ZOx8xV$#QbQDEa#5*;K*SEZBQP<emh;5eGcITI-
z5?b5Y=$o7@F3`3R-u)Ln+t;v25x;V4ckB1Voc3h>1yf63^<3K~^3LXk!)(ixVcPt{
z+)<Hs$Jw{?u2hadeSaTUUiuz~K1FjyVWo5gSiABJu`M|qt^A8#pR0d5#0jrGOC>jU
z=4h@JS3bPHD+<3A^K5XEysR5RVl}yvT4|{PSMmeycH%0T?*=#R_pI@j-#4+j8%dkJ
zWURePeD{*^H>HL8t0Asd@7>Lkc*D?#?VWpu*>*g-CGFdG%(h~SaQ23J6Z<&%HICOh
z>-R$3wQpix#@|8?VipIqh26G3vr+hL>dAHu!ahiPJ6%Z6&9M!1>>YYqHedBM?4|k+
zbv3LLJG|~oi`KV8+>Un5Y%K0QefWI2P%D;LPqWJgN=CP5xFuunJLYekeH^XrnzDR+
zc9x%6oJ-HMPGX;X2T>o_(&cI&uHOxDMa}9eJRMrO#F}<;wvE<p8@FExEmLQ%-zVJF
zv-#GSRNo45S~sgO>h%>NuNoZ9y#WcY3lnedYQ^-fN)>?y9Fozh_Z))Uuh)Xy{XAtO
zD&>mRzZ(>cZj_+-=+Db*ytGX_W>sgG7q63Vo=vRsS+<3;Ph@M)MxXk)TUUeJ9XtP0
zxioB6$=*7+b9S~5^pM#l=f+sQ8sc7ly-Z~{9HX!5CfbZ{q`P&Te&se3AF{V}cE@_e
z_X+cD8Q1Dckb9YRXVqGzQdU%+M&;`?dQLCTRcT~uTC2W4v(X548#8N9)bE5iSz`mJ
zmH{rAzP`P(;}2X?_wWXK%)Pgb_Up3i1NBOXyVywEmf2dSy)Dt&pt>i^dQPjCxW1g6
zk1yZiuO)8r@pNi2xj>TU6ALL%=kic&D(#0x`&7M5L;c?RQ)Yb`K8pOF?lt7K9sabl
zG4c@G>fYLPddM8MMa-_L+O`6n?U6mZRl9ewc6P2dhHFhpn0I$&O+$J|Df(je4Zh}G
zKUyz^xI|;psr)A4hEeZk08I%F+5Ir5<0?+;E|l()w}v{(HEmJY<I4lj{~!7d4G07Q
z5C8!X009sH0T2KI5C8!X0D*^&0G|JU=&A$?fdB}A00@8p2!H?xfB*=900@A<p%cLG
z|A&qg0zm)-KmY_l00ck)1V8`;KmY_l;GrYHe*f<{c7daRc!2;2fB*=900@8p2!H?x
zfB*=9fFFUzrH&ph{JF^dlU?DiS9`*P`W7AL4l3qGGlS=O3JDJ?<(ev)U+9~Zmt@*u
zhb|1#_HAoINf;KTHKA5ehozz*7Y2tYQJBKYwMr(pL-Ag+B4&(@j0T5zBR{q`)Zoh$
z{XR=$i>|UXEbH)~9%Ij(bS*N5_F~4;&@6@{Ej&m&B{97X6&>li-fTkzqhxvk!h>v6
z0~Xs=s_>v(oiz$eA*xc7%(zuaRcTv0mY1$LsZLvFC{nd5v1@}LleUFor67Gm7j`yZ
zE`EHF6)=3Cz2Dav(A)i<`#n;0d}K<P9G{#Ti;j*8(qt|+o*#=&jmN~P>9MJ?t(f>%
zzwlM@>*q8)|L<qoxCR0s00JNY0w4eaAOHd&00JNY0*@8}Jpcb_)h9{=0w4eaAOHd&
z00JNY0w4eaAOHe>1n~R69~E2!0T2KI5C8!X009sH0T2KI5CDNkivWKAf3)for2zpD
z009sH0T2KI5C8!X009sH0Y3tm|NBwFH4p#+5C8!X009sH0T2KI5C8!Xc(e%U`~UY}
z<>((?AOHd&00JNY0w4eaAOHd&00JNY0(%nBMmxgk@YBz$qoawnHAxr|N2TGMG&Mau
zHkDr+&W#Fd!{hns@rkMFacOj9OevSus+3nG^_sMEb}RPLTQ_gqo4dJCytV$(h4ovC
zIw>yx^mqQN-<iGfdms9xA9(&3r@sI4Po4b!FOGCRSNry<r>>t!{`}-uMxrC5!_n#C
z=oBBFJRco7KR&hVrE@IPPq=eszyEji{}xC8@B#r4009sH0T2KI5C8!X009sH0T6hX
z5jfSE4)5nKL|@<6xTm-`xIpsgpC5gxcc|x$@E?c%edto>HwAw+c&7U&x`nPk?f>Ah
zFCFXY__y2}9VfXTe+WSIeJ${71Cz%CTCZFZrS0VGYNl3|l#Eg?NX92wv@dW!JF}dK
zrxJW}VK%YKXZ6@xRvQTp96ugt^r=FwAlb3)58Tg~$x`u4a|u3c2W0uvJy}`IlDt&n
zN@AJ6zMPzoFW=&?C2sNYbZRlVKq==F3n_kKk^WE5&GiiN*|M^pai?;J>9X4e>V;|-
zNVks!RU&0(sf4NIe8R~yvZ=*_O_WqgC>oUXPqJuV;C{N5k{y^aV`urr1wLzECKARM
z(k@azuN@B##N&a+eVu4Q&MAVjlaUHiQ7WnIAkC7LSLJf4eF*pWyv*CjJYXeQ%Cy6|
zN%rJath<UF_1o6sE_FfmZ-&xoral=&>g(F6;J}F!f!7CgdKF12%GD~-az1fC=B49=
z9FWGYmL=$QmE}{3Rj(me^W}=<bA9*X*D7K=LnSx{cTZOQY~pe}J(uE7ouUYWsw#4h
z&X8@&*c(e0$<pLCSyI|Zixjk>;K2CP0qq&xd8C&TMvE{SXoj5@%fN6yZqiOKB$v{*
z_wb8uGz*r9wFtAJ4`rF8Y-B|{b7bJm(}Ak(s&wo|EIp%t)Y|e~QGA6*>J{yy!GWir
z4!j=M?S&=Kh5c_{8)4TDXdSxxt<^6|)x08CSVOiTn;wo1^koL?W8QnYKh_*QmCqdJ
zo3STRu2u@dj$WCTXws6T8b=Dd%Vu2PkVP_{R^4p_i$=5)S|m8Iaw4GJvMtAHgN6~w
zV#a7V8NHb}!Q4M=YferiUm1o0IFXo*>Gna?hO%Ze9m*QbGE&w~2M4A}$9~)1oTQd~
z=L+|wR^xL*t@yjC5Ye8|J`x;Qd@i6Z*;qEWGF7##NEuh7s#b)&l-ZPatU&IIZIM_}
zeA64+SW#M<oSqMnp`55!>eZeP4m|f<ps}vE6DyVVm-|s~d$Fz`u+5ll-yRycdk|)G
zhHkC4&U9_Px+N59%?|HPwTwu;Tl;Wu;Mr#bjf{@N_$7!An5M?2R=I!Rg<)QPn^~|U
zv-g5wdYl8hh{%X`^2k8qc!1MQwtA-^t5Qa&sb&3phWaXjnQAo?vp;e_Qa_^&>VwAD
zPU`gRAo~ON!(M`R0ApwzL~c~Gd(7xI64sozsQA{{(jC^W<E<#=E$aE2`#feTQYo+O
zRA_*xmnh3$np?c&j1b?PlN~GwIjJz{HCx+Vo;nrLp4LtT2a;z3THJOF%_BO~KhT-#
zO(EP5wRH|nF)bfKG7Qhq6z*{lEK)?fukoP)tvep5Uu>(HYryA>8Z&I5;u^}*Sg<KK
zS0ML;`!2$MGxIH=D@#^ZM_c9uNniTolFtCp6z=JkvAT9riv|bMG^={qhGGx(Ey(6X
z!0H(6aPFttmer2aMut%)JC>)q%<QNHR*x6i(9Y5DjkFBvTC&-koz~33{?7fRsg-N$
zX9t>*+h-r4N#a@eB+*W4f8c(?%)p!^+5xP+AMzwom(6V4&o@|jd;@oJlM(j({{Z)`
z+_7Kp`=P$^qyPBma_?{V3O#4TuZO#WzaJEj{8abd?xS5l(Dg#+Pj-GRFxBzpjxa@k
z%U)ZJ_lLL{tuNd+Du^Pl*D-HnJ!?YPty*nP?dPf&njVsfogbgp#v216uB>IuoNFcd
zPEBf7pp6Z`UB>UHc(w7q5Es|NZHO5jb~j+wprwVE84&Tywazu34sp|sGnB1iUwSR^
z%n*6Kcl5N3<Ym3vNvtMUQY+qD2=9(|;>Td8@!k;kf;QkGueU^}TWf0>P4Nsj#4dUq
zpx$5k<*ju#o(ge`+OvM<>Xyxho%>S4m=iH0XwgCs@->5e<2^xcverW2Y&)}aspeZ`
z-(1ATla$L{Gnah1P%D;rXC@~UR7o+6&Qd-;JIl{3&ZXz6I8H=sSV_Z2-p2)OG@b}@
z7hd&d<DN?Hk&!#<9$D2t-slf;=U$6ZW@htYWcF@3X5Ad5(HIrnGS7baC-lBHp`B_R
z3voBKm(1$&_-ozDn&#qd8$s)Q*E5GV%+!g*+NUX>#wE@3Z{N1cHTpu_iuR<Rs*d2Z
zrc_f=*O<kLV-37^Q_2@t{r!!jA?`xsmSb>^)V<BliF?42jYZ6NfMP=QP9@m<rQMWg
zmk|x6+T~uOFPqOadPAJn`7)s~lJcTZ-N=;%Ma+-`=7o~TSA>#Opy58hrj(1kv@KUv
zxwK9}rLropn_)DGR(DHRl-CtKqL1Uzo^SMoxR<mQ(?E<-k99A?^rgeB{pvRmhG}Nn
z)|#zU)=0%p#pbhy*;r9LJ&hfY%^KLP6F*C?pKF9e+{M?r9V@bgnf`YxW@=`Ph@1L-
zEAj!2$Qr5mjes5fTg^x>d5hya{M##j$ZCyHkb8;skcz}~BpWgBE%0PbW?z;Q@|(O`
z=2?XJT#9~FGA9&v<Sf6OScuc_J65m8W&){2-nh%`V^dp=V351XI<uC1bv4YKDN}tO
zzt3D*QX^kIRn{kCp>ZV0Ws=N7%9S18;_Bq9G`2R2oW9O}zH&ufNu+cZ{6&J~lgFnU
z-61ZiJw=(YS%SMjx=<UNr^mLhW_kDcz$}ydcMjKW48MUwrO_4SmVE2ftqPS+79SWa
zRjf~m>!%u>A#S{JpK;ZH^yAIjb9+1QNBBOTUyG|p8UZ#naiOwnes5TIRNY6pY~x*j
z%g)!M8|(jPx&GO=Y)TKoAOHd&00JNY0w4eaAOHd&00JQJND$Db={NE4i4)q+=;(F2
zJuImk^gubgkx(tyl)RLTRn(is?b|mJ6(yCrdPBTiP;aD`ucz<D@1~;SVlKVCy0Tii
zeIr(grV8T1LV7%XlfrLYzFkeNR<6#^%-^|rL%b<%Zm*__vD>$IDys`capGobeLJ^s
zW45}owYt7=CN^>P>fGI1xlu8dPhVSVx{2hoH^Aumu_-qH@9y~!*Z+gPM|=K$PmVs~
z1p*)d0w4eaAOHd&@YoW#L_f2JpMS1zbX1`AMQTwf3F~Z?Pk|mRm#W?oW^96<qV&0R
zrHjcp)g*%+o1leSdE?OupHx1VGV}vz=-l(;qob8VZCx&%t<-V_IZr?CR|`T_&6tmU
zI4M)co><Y?ZReV~PgXSMbLsQ%4}>X)@xT4FPZpHT@T4>*M)MQGWOQn3VnUi6X0J<$
zE6D}^`jzYa_4K8=<cz)!lpeI7xuysA_1rMlMV;ptD^iK#oqeUsZ;hReMbF0idS=Q+
zwmkAYzoDv?>iKi$XmMSsvR-+m${zdZ>*?#+uFNM^E2-7h?fJ{s#uAD6%zPqwcj=P6
zm|oi0x{{ln+bW5P=vrb`SdLCdr|z!ayME<zWi5I=db_XZ#jDW?wR-1FbVFWG-g>bz
zc{8<e^+w{hn!j7VHJ87>d_@qZ7nknj(&>rOONpJ^;(C1Zdg|Wj@@28FXZ%cRbN1$K
z`QB_^xN&*oUT#(#k4AS=ccpEml2pWG>F&(ASCsYY<n5hvvGI+mYw_{v)z$Q-SWNZx
z+`2T8T0XNFpI*K;pEx7VTrJ;QxRMrkZr!}Skt-(FuV1-QyB)n$$`>yeZ`~Z<m@6)9
z<x-O~Tk)l(zMjP0ay%JN&*){pduwiHDW8nrO(rfctzV0;FWro<C*tvq%}eo(>TG;t
z>{2{lzM6P(d|~!V<!)b(d?i;BZkA_emaZ&~t}K;iHkWTIS0+}K`OVeEYiqNq)sgbu
z>9txpp)8C|T#~L8=jO-OVu{V!+mv^za%XfUF?D8sXX?`UrR0mFD~oxGb59;!8K1oR
z;+0F{<@lMq$*b{OH|LeB3(2j!Q&*(bl60n=F8B2;+`AW_5x3Uvtj2FAD(7CjJioLt
zb8RhAzIo+pVoDsjRlIv^<jm-m+S-lfRC2tqRJy(R;#^{C?%a)hUr%ghe10{t^2)tR
z-mLeer{2{USI@1?jxQBQx5jQvkKCyU$=gfoYHBrFCFc3d*I&7HGydY8>QYQjEGV}Y
z&TXy7?xsiMvxQsn)Ygu;xO8i4GqsaCr{-q2wnr{krK#(4Q)BZ>ajLOv*On&YS4Y?9
zZfq5$a_zR5j*i|c-x!;}C8VaV<w{r6o41#4Z{88_jVSVBVP|C_IX!b_w0x_$)z?!l
zzLJis_txX<rI~z9S-UcKdvkMsyHLC}H5*?lU%h*0DRJe}{Iz&|^K!f}k&eZtX0F`5
z8=qO<B<5?I(RraXle}HaM_-8(+Z%UBw#7MR?Do=~>*<YJPD$@v&fh8AEZ>|Gm&!}R
z`rXB;>QpUP%<c5`T&vB+r!U{Uy17=XEiWl&7N^%=+?>5JBi$Pn<Z5y9Uh(RsJIO_P
zbV^u=ufKA2G*_M|NGmHNuPmRVHa5O+N4|DLo*Gr<)!FN%{Or=L>pR88()`BFXkl{Z
zMtt(_E0g0dDwnsm3d_lJWAkyPkgMJvQ7@IKzGM0Et?8|EV;d89r1-^)eLWXv7jnye
zJ-P!(EX?la9en(d&!w9k<oixco){f<ZYvm%b<A7Om95ZoXf=9Zu2><bN=w>Xr9^!$
zvHkzL`>%8T|F-{&{a@(cq>p%k00@8p2!H?xfB*=900@8p2!H?x>_OlI-E_aS!|!fs
z_i+k3<q7iK1noN6O?NjtcD=XRd4}SQx5n|iIT`qHH{CG^?{UAN@7NI6b##^M`^mnU
zqrZG~)%tIbVj>s>KmY_l00ck)1V8`;KmY`60@Y9t*KwvgI@;IQ7qGX}sGg^#>!Rx`
zZLe{DElR&tj!)%=b0gCu!(-!<`Qa&1TpOMibK=@$ej*wj9ryln85=(zo$$G&279>h
zIU`e7CL(Q#M|&VzdwNvQA8xMIBb4iKORjen+Eel^%T(#6OijM7Ox{wwbW+@J?6*!Q
zq`WXOGC3@WQxn5uqm$9$Ty!crJe`}Eo{HsS6S@4Tw*u+#y3sM8OO$D5-*tM3xpIH7
z-#R_aOx1nX>7nJ>)Bj(%uKwTT`oG%$d;P!L|C`o-c0<Mo5C8!X009sH0T2KI5C8!X
z009sHfrphqcV|aOAk@_v?g+BCBkZl4lznfYi{6iQ(pxw{Z(Tjdew*v+`x4jp^<&?9
z>~i0ij{UarABEus0w4eaAOHd&00JNY0w4eaAn;BSNOsdt@EwQnNJZ#sH~p^O;d*%B
zz|Tnplil=Vxm(mhAAUIUVmJK)?v`}GCmLw7+i_}dPW$*BolTg3*Q1g@r+tWG)9%0H
z5#`HiAClPI{lzLh+;5&!<NsjIJ_ND-{{sCzT>tCzh8GBc00@8p2!H?xfB*=900@8p
z2t4uxb^=GamA+6o&DN}hjIB<^^SPQ_pp#y!=eK0(uB4oI4#KU{R+#zNXjB{)a}(n5
zSYC(?PsgT3TDKz7Hk|nhX)4#c%c$=fhn<dHvzA0zEqk)g#d0HSBa>sp<CD?R;juAs
zYM71?%@5}$C&z_}$^6>d_>|9D3ZF}C(jVx5gX{kWz2OA{AOHd&00JNY0w4eaAOHd&
z00JQJC=p=m0Ro+!;jl6P|037_Tm4^rlnRKlfB*=900@8p2!H?xfB*=900@A<yO_Y~
zuJB67KAueooT7-~mWZuS5@7!SF17}!4gw$m0w4eaAOHd&00JNY0wD0H5;)zN?%2<|
zgTSfIba+3j1lan2?ukbg7s>?!AOHd&00JNY0w4eaAOHd&00JPu2<V&sVg3(e009sH
z0T2KI5C8!X009sH0T2LzN16cU|BrO_qHrJp0w4eaAOHd&00JNY0w4eaAfOY#{2zt@
z0w4eaAOHd&00JNY0w4eaAOHf7J^`%%fAs4ZB?JKw009sH0T2KI5C8!X009sH0XF~t
zAom6C*f)-SuJ4!oQb!kh|7Gt=&(HVFhQAO#6Z*N(v%%bv|9E7h`}ewUcl}1E68P@G
zM90@UzSvRj_%QdIl<@Fg$;McaTa1MJMumc;sB&qYFO^k!P0kCdTrTkyNh!+JDt)Zx
z%c8^=<#k058HtXJ4o9blqf>lz@_aOQeti0BV>HMuF<GKguJGnT2>e>PAW90KS0q7|
zcp=Z^?k()a#z>Hx*M$kyja*q!M7J`0O%{2QQjzu+b+Hi(a+7UE?OCpwMl{IHu=J`V
z6ssOJY#w_``FP`8keg;wl$?;~MY$^E3X*6NEh~Ep5F2MhoYr?-uPc>^s!RZXrzR;o
znGLzBmX#e|D6o)xxlk*XOypuFx5KMD6^U2Nd`@1MOX^BuE-{nhqZfQ?QCn&Zhq(Lo
zFv~|2dATG?+umHxmX-C4EDjr0%H$7=f+|$YH6<@)sAguC6Y*4nPcF<RR{3mlb~WSC
zGGnCAIO#Lq^jUs!fzNu<WckxsBUyGR>m;NvO-Unug{pt5F%;siympHUWg4PKy2)}Y
zW@?^_Uz#KRFY^nFDL%29TuH68>eG{z#Vi#vmAH~v=C3a&=i|$__-lz<d_0|6OfHbD
z`NTp>m$8tZo9h|kvjiv0rxL5)t7gG$Su9s8MN{!w6T4ve3b4Wj)lt@+LB3opN+k+x
zQHvpxrD}7CFJUdvI1}QoY0sHv<2Vjp7D+t06f#WRm|CsFiA-i?MK+AU31SxD3ia2b
ztMO5$#R<~FN$J+WZm>p6G1^epiJRr)v$Oon;#_*3Owg5|7Y{$1xExQ<rT9~)c)vyz
zXhcF>Qe%zPk<AOLs>rz-xxoih!%o*!6?<s0-8dcMUeKQQDzQnG@PPTVBoCJLryHkO
zJv?bzgXxeqWRa?u`5~oDu1kmSsRwr1TLVUtR?D@E9`)rv-uOs}dr7OB`PWJ^8MEoj
zay7X?Jza)nX1tYzqGVcSdLg-#wp~|xDUqp4l9&;w4_q&?c9QW=ZdhwmTxPQF=;W>R
zOj&+ee*-?zcs|4>G=UU19Tk(wJNX;bcJouMO7wWO@R~AnOO94eVZKUkG(H^UUbySW
zmI*rR^Z>0i^_+As$Xq=8#vri^zFEE8I2q(t<|(UXsaW2Ucu!60!pzQuM9eNPUbi|O
zR&!0kS-*xl-WUvVWi8`Z7<biKeU>g$7IoQeO=hozhOxL~dAc_<57S!w!F;-Lf;u$4
zZ9BcRTcX)Xw>n#QB(qez>f>84+R+Bj8W?l5aT+C<4cw>;eW2lLWIheut%SG1x9bac
zZ}3}<;~{QF>ocp-7__)EIqPcUu5pPwie9JwC|Hg7*TZKUA7YOFnf4i*wYf({Saq(C
zeB4?w%di4{tD+ugJV$Q6!G@Nm%P`aPEZyGzqA9n73~N|MO&x^oR@{Rm&$vUM5KzA|
zOJl`Uaz5c5C{<|SBl(8<QrK#Iu+;!+RY}on^{oEMZ(wRi$(T_L1F$da-ie86=b^Om
z#<Oibh%Lr)A7&<oYqIreb(4D~^wH96I73-85393jr+BsTfe;thyyJX3-?r|<j^Xhg
z_Enc7^R1^k*LWtxO*hUs*2>cE(^TEr9K=rBITw9sZq~BzAlhnyzJcZl+s~4H8Q}T<
z!@Ge2Ngx0MAOHd&00JNY0w4eaAOHd&@URlV{QqIC8R!K9AOHd&00JNY0w4eaAOHd&
z00M_c0Q3LDV+lzh00JNY0w4eaAOHd&00JNY0wD0P5@7591O4C5_5TZc!wUpJ00ck)
z1V8`;KmY_l00ck)1VG^JBQO>Sa?f;(j)rf@(p^ajG;J0$a(;Bo=hA2gr3zE3gecPn
zY(6o3F0tSL1O0!&^?!rj@B#r4009sH0T2KI5C8!X009sH0T6hE2y}OLbOZvOo#C)C
z|NkSd|1bLg=n*O)iUI;400JNY0w4eaAOHd&00JNY0*@<!p3Y}F%m)g>ozIw02^jPL
zH`wq0Z#=GGP;L+a0T2KI5C8!X009sH0T2KI5O_ohbajS1^xyycj?}sLa=*;={oSKq
zJNivM9}WLl_=V6fhCUShr6a+vf7<yMoqfS4dOv^UH~as`{()mJa=&~u*}J1Nx#A%c
zSn0Sp5I-5vj_b#rvSZfsWkrz+f-09w8Rs~5*H!LcwVp%j3U(zjPMCCsv6CGgnc2v?
zmIw}vo_y_N+Ka(~;o-nHNA&ZYMWMQpD+`L4snl`>Ip6%9`*l+T<Bave=I}xObdT$p
zboa69?$bk^Wcp#kEoYbScG9CN7bQB@S*1gu*X`5I%>&w1saPR^Qf5m~WFc2@jz2de
z2>HBJt!C8nrd0Ad^3*zQJ-5?(X0?9Gq;alk#uILew$ieb+IKq?HRFw7rk1N2sl+s<
zLsP9o=`CW;;kir)p;RiXx{>TF38gTJ6{NgwTKmX{5>+<@rTJ7?Tai<Sb)1T5k`@mR
z#A1PZRkt5@==@G*L$1<tnw<<?Q(CWZ|H5NK+140jtOq$C)DYxvQTo}V?GCeTXKr?R
zV-fAD_KD!Yol}8&&~C3@#SI}D%M5Zm89D=3aN3+FHup=dE!h*nI&!wHIHQGo(zs4<
z9m+Zh=_pyRCfP{+1?`34z^PM#*DkbHn@1-1&v&V;1J;v&4YjKTX7f$@Hyv|M5nQEh
zX+BJV70%jTbMw-}%SCoPypKy@G-St*ET;Gc)?_2w+Q)+fFPsQy&)W7-k(8oLN6nM_
zmPL{#ugQ|)T;zVSwa%STR}!OYogh}Pj=*fBcZy2~2ZS>L?T()J{?EB@I$WOnXQoDU
zgz<zndt@MaGQjB~t9J@Axly5}mi6zRzA5JFSGZr%CbUbzftO<eO}4f1n2_!UtOJ@e
zMi7}mYaH&E9d(w{396{s#44X<#cPe+D$u7z=vgy-R`VY+y$v(L$dY#9$iVYY2Wqz7
z`a!`I#{P2uxUDu;G$Tj$P0xeL)IOq}4-TZC4`?sjQr$;%TkTpGVV*?q4(I-_wz;~a
z`H3;KamQj>nayg*K8D=*l#Qs`ba3DTDf*PIC>wyd)0#QBFL6I_>elWEcB=_5F7R1*
zLYE$qk~S3_n51lb{j+6Y+-bQl?~$p2-!Dr&Byv}q3=UkR3<uh0NR7{v=Res;p6pEV
z-Oqck_h=Krf#;tOG}5}4b!XxJ%>A61oiV6tMt6{|(16l1u40#Uy?uS)L63AN|7z8I
zvs}9ejYf_Ixut>0<L-*L4+OUwvwea4+147+=Co_Uf#QXL_Nrc!+C6P1IB@c0pfRoE
zb$KP%NACaj!gU3aS-J<S)^LmU2Tr%`(<RWTW?fP<u5Qd6#1({`RPae4O4Ym~SJ>#w
zCp2F!t;y?F0Ersn-Q(udygr~F^5_(aY-;1wj@|@Lo3#bsux9(R?$XRt)g#(TZH(L+
zW#_mxJErA~^b4j>BP+ARx&PCw0_tX({JI^djSL^1Y=>rMN5dP-tRoxRXmDVhv~&h(
z%{kcLx&Ncp#Oy#*o%Y#BXfj75-{Z#vjXvG_?4<Sw?q|G~X9wtZCu`TVE4p{6-_!-L
z0fV}ek+$fA`!n}zUNP=4Ly~LC;5M9iI<=Twpv3cu1sb(jBNwu6Qv`EhEiOXOKcntc
zng{=`VU*7pD)MRVa`2s>&jz$k+snIJI%})0uiUS;R=q3OXxXesab47#1MA_9-~YeD
z_5XSQS3EO%d;tLv009sH0T2KI5C8!X009sH0T6hX5$Nj<cRbS(FrWVK?GA_C&;N5z
zyvvFMbwL0GKmY_l00ck)1V8`;KmY_l00bV006YJ$H}oa$*pKzS(KmYZt4Hr0?e6(<
z&xP>!g+ED`@d5!5009sH0T2Lzcb<TDs!<7XuWFOwzR|cS^0kso%PDyMIZ1x4tkC}j
zQBuy*<EI6Q7fPaap+YObO8Rro=HrsB&+i&ZGkVgD6W@4l)(B(|PU_+Gpsp)M<U-Tg
zO0o0f(=&~7kc*!oMbr99J5v+hP^}=9*42&SVf|Uw<-}4txty5wk)l1_D22GPc6^Tl
z(?g-|;u;SY?^RSv<as#QDI|Lye8_qd-nV#KXQLS6E^5Q3Vont}U|OF{H5ofm+T`y<
z*h7u_3yngE8-48@sbMUPvN^LAh^7X{Vua*nopxe1xsqDpvvz3@vM7N8YJK3i-O>T}
zys=?7t!aE7s5c+McR?_dnE~bsu-(^Y`i1(|q|(?7a*{*<i}JccD^7W7Ta}cOP~e4v
zq^L!qB&<tHl~>C=End@?6I7*r#Wj}<`q=d=jaNdP)*oTH>hi6Y$~$Ytsz(02ps)3;
z@`_ZENIi-vZ_~S>0j)#Gmqp3>=95{nAqTldmX|<2qoT+~LD}Iqr5#ozXB`<^HMgfM
z?GufS5GT|-O=%tv<X#DrG1Sgf^W}<VhpkS>$-Ts~_byv<OEBC}Q*`E!+<8)CG1AZn
zJ>RTvjcenL^$_=xR^77<C7~$sIU&E<)|E7A<Sl@{n1RXIul7y?U&ZTljkOS0eJxCi
z*9@=Pls|7v;Pd4|typ4JT&fjlIZHu`&!rN}yx|O*)5T|J`I*JJ^!x&^7AqM?%#HZ+
z%!~2m)3NDj#9ji#)=ggMNndA+qg@uH*CD@>NHxj0=%{s}XE!Zomlv;FYS=35o@6<(
z5T8$2iM+a}7F`+pG>o49|H^gs|2EhE`~Cm1|KIihw)NjT-_b(@5C8!X009sH0T2KI
z5C8!X009sHfjtRyb@p|H*jtdj9bs?Xl<280dOOxhZ{YyFb+O<7@%%r7fENgW00@8p
z2!H?xfB*=900@8p2t2w3*#3W*|3A9*i;{r=2!H?xfB*=900@8p2!H?xfB++a-~V9;
zAOHd&00JNY0w4eaAOHd&00JQJ=o4V`|EIYfu7B{@X5YW=D;>@E{z<RY^JhIrLO&n6
z7W~oR#qPiCp6{x5e!cTkfv<Obn4%uei}vwJvTbc<Xd?%6+ZJO38FoOjC{#CcWkC@$
z#?kC_U^6>alpSTr5A$0iZTF(sC84naMVkord5C`d_%v1;FNV04&#^6smL*eJTM%!k
zNSjSC6&>Y#cbu_Pc?(9_yU%WQpL^*(>C#Dkkj-a`Ql=`Ys$5#HS{v7xyUD0hu~MLY
zRWe(GBC~BWnj;ALyi~1b)bgg}d*-Wg)@~-Z<8ziKZG2;F)FFDqy*t@t*jeUmFO5`^
zwz6;++Dhz=V5XL<8L7lHcAXk(CAPQfi1zqyC6aAi333-@s?=GfTxqIC{SZ@qFAlpN
ztzCnhTJRWW)@OGO?U}~qFsJotLCS^B;jWnbRI%f!+laA_j&*`9qt~@@9ainc@aW+y
zz)!haBN5~_*{(k>&Gys!+=}cG-!EV7Mq`$sJgMEMtUXQtAXs9pTTDF=82!-eHYu1_
zun*{U;;_c&Yf<FWW)AApta7w&5aeX+t!Lc_uybA8MA+4*)#Hsz<kMbzit0ew)2CU=
z*=>QfvAx>EJLJ<`Cj%?enj%#<GHgq$R?lSnhta6MJJ+sG(6_!zjd+l|WqT0Xc#gKm
zGJCGK%Ab7Vu>48P-Hac!<@vSiTH}QS`4f*K2k<Af0o=z!+@fyEdv(q*WM4zp<i^Di
z7q8bGk8GZ*tskIG8^7(fm7Y7gKAK^><0ThTi7T|<mU}$&_MYSIH=HXsJ{IJzu+E3=
z1?M^x+Zw_d+lC1R+5*ogc+=6@u8qR7EyvySe#3wxjSFO<+6gmHW1#EGxq4RLQ%_<8
z+}8fg6*1$)Gsd&7Fy`-F!M+uJr*WR@S?g}E$etS5-A+^1?7DBB^<ZN<#LYBZ8#KC-
znuND@ZJDn4x^d|`v>f*CDtzlEi>jckOZxVoO@m+B7fvcz*Ib>4&#sJ4sN14X(*T#s
zPW$bZYf4_P!z>^1MIkn(XwSnwww0&byl4YcWhb*CSJg88t&*x4+S-uJQQy9<R+W?t
zO?=oX_G~Y^oV+fV)Rn|sVkX5${VZI2Ph*np?#FiKG;CZKWDI3!2+Fr+=vF7&8l2_b
zM&E9eOS?yn`L=@5#zctAXqz76^W=5bEbFjwm9Vy*>3Px_%*@0YJF(G3T4U-XBZDec
zn-kr)&a~T&@esFEzd~x8-Ak*)eOYm9vIBZbN8?sIHz7FA*P0IbPlE9L|KZ%ufD{k_
z0T2KI5C8!X009sH0T2KI5O|mg;Q9ZDxptr%2!H?xfB*=900@8p2!H?xfB*;_4gviB
ze>fZ=1q46<1V8`;KmY_l00ck)1V8`;9%cfV|3A#N1KmIX1V8`;KmY_l00ck)1V8`;
zK;Uo)VE%tN93ce+KmY_l00ck)1V8`;KmY_l00bUp0$Bh5FxL)r0|5{K0T2KI5C8!X
z009sH0T2Lz!y$nA|KV_i6c7Lb5C8!X009sH0T2KI5C8!Xc$f*``TvKxcAy&wfB*=9
z00@8p2!H?xfB*=900<lo0oVNh-;l)Lp#OjUa3n(t2!H?xfB*=900@8p2!H?xfB*=9
zz{5hoHUEEDRDvEL00JNY0w4eaAOHd&00JNY0w4eahfIL2|Nmge2e|%^9Lx0mT;J)V
z-_rX=?{?4c_k2&dGxU1!*Mc8C@;ygZk92nbO!ud{KhX7kUDKUE)|n4{Bk=bF!yR9u
zgpce?d#WLYIIaKHaNnpR70X)^Un|LXY7#G(L}{DP)#QRImoiGZAQ^8Zp(yb=X|1eC
ze5qC_2)P2KsfdCq%`Puq=aUPwiB&$EUP@%Dk|bt?YE@n@6{V7z@fVgIiH?j8N2iCQ
zQ+#ytd^C1`e0rxLhPV~2le7{<k+0^<6^YN63$<d2SIc~pV4<K&3a{##WShdWe0+A6
zpIMws&oA&<T~L-!C00}XLV9kF_sRR!Mn1%)wQe(SA+O5i(qUw7NZKR!g+?yO-HPa$
z7v&OPE){n8wQ@m}lyjm`-N=;%MdT}zQj`g7xs)%95?_?p6@jVgQyj4&Pz84~iAA9#
ztV^OP&LyPE3k6A0c|*f0uSgY%>Qi(C@0rcjMmESTv0MyW;MbLMtzudc8H<^NQY%PQ
z347?QbvH60?uvGb#Ko!p4O?VVYBKxAR|>*TL9VJZ%ZYd@!KW9JOX-B$^sES(va+6$
z#SEp(@{42vRzQ|Noi)O;Ls=FYx!?nDt?{WKmwuVlNb44^=Wp8KmQbikRkt?j>$6N7
zD+_)lkzxrhRus9UuAM&hym)Fz7JW0SHC_&JajnlRa5_2bvN%hi$Ih*bUIW+5X_)t-
zY2veq%klJFijVpkd%W?<Aol{Rw&hFl8PfoGy^R}2RaW+_t=o;4g4~@4GBDd<_Y|IJ
z+zxVA+`=8(uRef%E;nukxy|ELP$O^VPD(0yrK$^bQ4-}^F;^((sEsK6{LaeK-2dO+
zwf?q|U3ZFJo3gV;x<D5~(rz=D-C!*&xl2(`D&3;8EN_&xCCBov_hFYFaxSHThMd`%
zp>0_af>||1KJ`!NhoUIbFa27yXg?GM(ob#DHhrMYrb*uqplH#eMFSM+xpy8Ml0!)&
zk)lNY1V6-?;oLjFd*|G9&-tBmHdbz}EE9Th(_?x6y-eyhzp2d}$yA&zeluOXihN>J
zHIj`&E**3icYOc;c<N1ep6=89hNP)a7rlFKvz@LJoqwTJV~$ed&6l=FM3m@yZ?cCc
z`D_o*(}d?&#Lh$amHl_eQ$^RNzEmtO1jIv#7Q83%dvb5-`Yn3nZc1x3&sN@B-MF_Q
zJ&Lw>CDNcTaQzv1B`b*zbRHG2R3IROtK<9m#{N5(Q*OpB#)4+?<;IhmO<bg^VmpEG
zvzTLPj2iTx$6vNqZ@gC$Phl#t_h{nkK^lKiY?pWU?~SM4c0V24VVXI0#c5cIE}7d0
z4|a1Lg+SxIZ5`g&*7qm&H^x(Me67+tutytHy1zG^?PEhD8R<@o+~K3#6~vA!gO~A<
zbp6))bup><`pj=ab9*zU-HKA#VkQ1P(oN#dd#`s%)ojzSHNHB=T;(lRv^H?beQ)*l
zN=wt;8_?~ntvw^;SJ&>XytQ(-<$E=)#!AkPNvB(%G<Im-+P|Agz0F0O|Hi4Tsx)+3
zj`^aj)i?b)dxUlK`TZ|vQeXWl-KwlqwrOhU604ccj%07qd=4aCRb#qpOHVYKbwOK3
zRWci8U90$uSg321yu)HK6L)#@Rqx2F()};4++Fc*{pPE$#w?#-*}pTM%DHb)e}ozr
z=|sy~*4|j_nkHIpLsAM>wXicW&AXr8Ums7cyVKOZ%5+A;%&^QYrB0fS|5Rbi_6?Hr
zw0PgVyM9{=j|LC#J!$hWEwf_4SJ$L#zG&x%TE9FQ*QL+yukkLe`j9w$rr6Xsp=Fj>
zohz}bKTruUioZ|i><n>>)dt)j-@iScT6LHG-YNCE;fU4DK7Q9?Zj!iPKe2yHNc$J4
zV|NK8+JboMesf5?0p$Xrs%`R3Li;HxCp|7K-MUBbs<(JOYQMt&@+0Z)%Gwf%f_v*y
z$2D=g6?b|6ZR*~byGh;i-(gQ><-4FRt(25Xh1qt=F}7J<^sTGdKgu1vg{jMxe0_*E
z*JbU0cmGTD7%#Yc?T^7%#V)rF6zt|{{vvRwI~UgGH7$b`dy7M#Csf{>cjd}&tb~RC
ze`pODcrYLU1b_e#00KY&2mk>f00e*l5C8%p0#`5nRO*G)QtC?O^0zKu9sjv2Ke+fC
z7qid(%5$H+@Js1GPfv^~sin+!Gh3H_5PqfgmydtaeJ?Zdna`wu{sm!o@>~nvzo{Si
zmJZ*l<Q)|zbMABu?8A5NuHIg{`@Zz{%KOsNJNMRC*XV1vSJuc&7kE0z*v(BCZ8l>%
z>f1WPdquzDXbyJ(wAz{JJ1Y5EEzOMDy?ukaZ<UWU<)1j%+<kzle9|qr_cIg9YiYM4
z3|~k0{<pd7Q{RuXf*t*K+$GX>OX*sacaW6?m4TZOC*7ZR-_1;HQm;QcTCY9-cuBAb
z+FD6{^H?2^-th>1_b$(RgosIZ*?lK7@!D(YkG~*hWm}ipKBoSrWuD4yr)Bc-T)r&P
z583YyowQlH_Hq<gF#*L<uaC{o0<Gr95PorpZZo;#-px#W;ia_u`EV2h=XPKkq`4X#
zO8s>l(1LFgToU<~XGRQ8a64#dHx$-F_mcbN%*68N((Y{7c<9A!x(>NZH9Va9Ue_Lk
z*R+1M=`|FNRNjrI+syaIy3e_HG83Ppp1k9Un82|c-ZS+{%K-Mt+e$vw^~}WW7t`*%
zK;$rwDDPhCyME7l>qS-Fwamm6_2g>n*o4@Lw*Ttj(FiYY4niyblUIH<^?R9#TrU0f
zNr8ORpzN8c+drrNq-7lB_VfHriU{(k;ExLK=GP*UII<KREcAuj$l=~Wn}ol6!Tpe?
z@n>45u^<hV#QH$fQFyAUC4bTS`|ha4f6s?coa6v{>rOWC@rt+UBlm;M#60!lrRIQn
zccat(Tj{s|cZap#{6>4rys2#P9#4MNeLpj?LT$d*oUUG*&1Y_H`fvTFqm#AEH`<$S
z-ZIhy_cixcX5wKk?cNU`yE(Y6u8yb|ru~}KUv$N!{Zijk)#5`EeT|!pZLS~pXWX|l
z6E*5C@ym$gm(;n_-j~#$cH-H7P0(L&PDgx6d@T987u)YpR5x$Oub*^(%>7bk;%$0@
z=|cghFEqdHjYaCu661VeH$|U*)?K|cv5`xsM33x`bj@KUrQsOjbL5Hr`7qVdtM5F2
zKlKG6hkoHxN0dXC-1jaCMfGFLy`7nO@x}DlR>H+Y2*<zlA2rZTJ+cVM9351T<B&O0
zsSCfBnfUzY(;wd#s1xZ(b9PGXG5r$#J@v<}qYynuqV&om{t_Ija<%Le{9EuZc4+>h
z{aE@5Z3{vjKWHypk|w|6-pouay_|MG7s`c-VOh-Mvz2V4VO1DsQbvxZ{;2cikDQKt
z$rGiKb9@nL-t*Dqrn{1vn5O10#5Uj7?zazaH+W$mwKXW-)j9XZrHR#-yS=NCuau@D
zA5!0PpSjDKiI-na?-#{@M@~k5P5oi(a7Rvw(H4E{7_|r#okQv<)+Td27OVG7f6GSn
zYZk|TirM`|R11$J5lkJrRaS~>Dc8JxXf`@L(&VH&=U%6EZYu3+VubjvjF{rqPHBs%
zxCZ@iQ$I@$n^~N4>z$&9?Yt)S2R=fb#NEz|J8$SsrOqq)jPhHj<vDEs(Io2swVE^V
z2?ziIAOHk_01yBIKmZ5;0U!VbfWR;!0RI0ljw+rB2mk>f00e*l5C8%|00;m9AOHk_
zKq~^+|F`0SPe1?&00AHX1b_e#00KY&2mk>f00f2+0r3Baaa8e4KmZ5;0U!VbfB+Bx
z0zd!=00AHX1X>Y*|GyOvd;$VM00;m9AOHk_01yBIKmZ5;0U$7p2w?v|jH8NY0s=q)
z2mk>f00e*l5C8%|00;m9Akc~c_W!MT;1du40zd!=00AHX1b_e#00KY&2mpa$L?9CX
z|9yJCZw})#;+cQ|5C8%|00;m9AOHk_01yBIKmZ7gIs%dX|ENQO=LZ5n00;m9AOHk_
z01yBIKmZ5;0U$862!Q`Tv_p#r0|Gz*2mk>f00e*l5C8%|00;m9AP^#e{eK7regOhN
z00;m9AOHk_01yBIKmZ5;0U$862w?v|v_p#r0|Gz*2mk>f00e*l5C8%|00;m9Akcz<
z@t9ebrZQVz{P5vkR^Fr!cMSTpD{sk*3t9Pzyf{_J%1`CRd{*9}f9WDkrGr`7FdfaP
zi)(eZ$?9};SFR{}h2Kop728=eoVA9o%MY`%s@bNl?A$g~CNIj4Qr4NAmFZB0)mYu3
zV`al}j2a!Zw~QyZ{2Mw}gHel)Vd#!#@&;<&9c-<>DQ;mK7XNnLs55zwU#eJ}%;8<w
zl~u)2Y@=aSsLfs3*)e(Bre)Nav&9-VwbkIyA}fWtnJHy{I=3)eoXXA2%$0LRb*`A3
zUzjVha&=~Up;(mZ-b~dT!}6b@!aG98o|+cj!fw>AS&GB1RgE2Hm2A3B*<>ZdEU67T
zPC%6^Mx*Y0_)4k7uPbSFhgpvm{o&-KRNj%i?`0*!%u4L3V=0q#Z>?@S3J0gE=r+sB
z9?UlltETX0-rW_A?d?h8f5~rD%1OJshDnEacg3yt_UK-gQs1P<=YY}&R$b)}z9jF_
z*f^R)P%g^rCSB*F<Y+Z!GfShHv6t;Vl2wiA>atO<YMXp$R900QIv+KfL56M+qTxuJ
ziq0H|<tj9zY^}n_q0*p7t?_G=r<%P?^M_eX{eW3U2$eX&?-a~9LV})-AHJoOnNE-E
zf4mc!gG$3O=x|xFmUL~i?#)b^_^PEm*>H9QO1effEh|?wU0*kqistaQg1<S6s!>n;
z_QZV5%3Jhq-J}V}`x?9?{$Pf4+W-7md*=6^o{&bKS>oj(OoGv;hFAHcnuacB6g>m)
zjABuD_%HW0)uFeRUQN@W>9wIfqfwou7m8j=c0;orODi|{ROG$!$JxIDHOXJQJN$dp
zqU-}<n@XL(!Oc$4gsg1SyT@;_X&DWACutUCgV)k~4QUo_S&VL|tF{kYIGjecD&EtY
zVw!Xxy7Py83WnG78~g9LQnn4f;c&EVhnf7+aC%VVjMeE3y+RFo%sTawPgH;Q&<t0b
zCl~U2T(~vr8h>pJx)Xg$Ao{ojjL1Iv1lf1E?6Y;_$@vn18`NV7QV>#b1f*b68R)5h
z3nVzX$}|lT*rNVz(dufD3Q#M3)F6#b)7iEB>$6$u2x?)uQQ2nQ^+KIxJ@qcC8%|2J
zxE-P$nzMPK3HCro?1KV$fGQn}WwF0lay>0nv<?Hd*c)K4_eHZ#2JF=9`7;D;O?ldr
zDZAXZt&wi{doX%yV)i$rf6|7CE3Bj)(j|6fi`A5OnPm%Id78Gtwy*rFinYyeqJ)Yo
ztI;Ob@ik(d!~j=lNlNk!6xU^|Li?xv>66@ao^Nkyhrze2Pq+@sFETZH|M`Z;|NM;a
zWJN!Mec4UErIa?jEhXK-+q72mtdO5wDCf$BVj(v(J738ysA`pVu4T14UzyA2r)TM6
z+TeSSL0an*#<n+TgUq#Hd5;?c`l2o;vlI~et;PWXB5VV&4Zt=KZ2e&y5VnLNunml?
zw1YL^FzYgGqD?zLY@(zbo}x{(zt(`^7kaUKR_y<e<w}di68ry?`zT|8#r~fpABm?C
z*#DDTuw(!KFGKB<kD2c<${v&75Q`;*a$kgU{`KOSLbli|qwK)_2*bJD-AHEc?!zby
z+17Omd>)^Ia~}9SN%yWkn<sE(&%l*^5?6MIT@U_jBB4d8p$S!|-T^wQPVEErr$NCx
z8cNiF(Crm*pjDIF@aTYtum{eqIz{gPe;X8|HW@_#!2}2<I3O%z$aWvte4L}r2Z9Ms
zCy=T`u{iV5RKE7W8Gl~j8A7TMQbkPCVGOBCYJ&09zrX^6V1gqU3_vCTnLz0t$$J)p
z2@bToL?fY(Z6Zu&(Haet%dvsw{QpynkcLYoh-y9$E|u$vL!`YGeQIvciw~rzbA(7A
z9wJTeZh6OH_R+IK93k+p_h})dds{|hbIWY-6bjhQ93w#l#NT-me=4hp*YbF2B0?cJ
z2O-$v2<5@hA{seNkK-7QqI)7Rk_G1|(al|i9Lc96B)SoqQ6R~VmFfnP>^w;_lTq@k
zx+5~*kZP77=Z&Y(6=r;eZXU9%LzW#S-xnp~dh>y-8^(6Sd?5R|z39l6<~)%14%Yi7
zr8lH&N5~){OyMVM=(VF{syMKnnr7RFKhX(9`?1rSeNl6yyF6i5yz<(@{-5PZ;f6E*
zpSENj{D0d2pY61IvD_(`LSYJpDKt^W8kj=c(rtE6tCYyb+#llEx;d<zvs1G-gG6(V
zNHj^^$6)Xun2R`}($qn%IktQ5u*7L2l98@=fF@dthA1uFfu7_+!8;l{)PT_Kk0P~k
zBnE$r1z>3R|HQIt-Tz;``s=Cm#d_-UFJG+FfA|3cKmZ5;fxaN{jmZn2N{w9}ADbS#
z`t6s}=|a9Roy!+<`2{IIzc@9&n4kU;)w8JA$tBvXtK#L8-IdD?O?OtQa06v=iLGiv
zxu>g2U7^*>%MP2%NpE?Fq+E{su~}WzT=5Hlc#5T03`DCK^<}T<hHPz?m1~p|xG*(E
z|ICu5b&3}vsVh90t-RFio%EoQ&ll#TC7tS2aHF|T3sDt&t86HiN;wI5t5h*1+HO_j
z#VFpVniV1s?QNsZD^_@$b#v=pQ09dW@GRq$vX<&oZ2L7LbnP))qrAC6QcNOJGhUgZ
z4~1S3QG`@{11?WOLg*j9F_tEoF+P5oWX8AF+eOCgVtx)HqfKP+IxvtLHi{ZiRU}$}
zhNGwv$%21OrP6m(mpxhV^;)a;kS{Dw&p;F$TNIp@0bp*6D@uv;{|nAsj>>l#R8lU~
z@TzmHqRbWMbBek!mz$ZM&*#edg?z49o+~a)m8a&)m1(*-%>Un!|L-bItFf_{Q}hQv
z@d*6R-REcl9urFY4{Pc4h1U0Os<@b6z<byB-a%{Eu>SI@ghR9bo`EVUNP&VBC`f^V
zda9_W+FmYdWNJxwcxI6TrL8_U%(gJw4$5qsl*8u5*9<BJ3T=1)m2Y|dTJs_-BN5rC
zKYR@kUn|TjIv-SEM=APYE=8YIXn^<%;%mJB-=s3L%-QO3GQL<QarfF39UAJfnZ$|K
zA`o9oJQTG=RqL4*DI43P7?&c5uYE+fKZ?}Rh_B~1bv43><VA|-<y<OugE(TAR&H?8
z7%!0P6S0UmHEK3^WUcD)^4~r8K(ee=S;>nGs<1)C{~waPs>S&~#PR=B=cbMSPy7F|
z=O<G12R}dn2mk>f00e*l5C8%|00@jG0>A&k#peUgtNV+)=?l-bl3oji#i;^FuOPi@
z_ARB%bczcknz`~6q}Ov~a3j(7&s@CqeEg6e_N3`yYZ{qxi5=o>S&YJ$bd^F}cY~WM
zW!umj4vzw{9Y$IDdsDy>meVt8IUR((lt?&-m`vd}QZ41lhO?vdsCe#aq4%+>>H4~<
zR5WLY!t}k6c2%R~%eQ&r;6+)~{m6pBd_)HGAjBZ@{`ZANg1rB<fUB&kG;~UpLBIR)
zMB%D#k{j1?SgxWgwyjlo;8dkyJ4TIP%r)y}5+lrF>IclC=+Ua8+ssqBMcxiyz&PwF
zug}7-?}K6vi5BQ!$1UJA2#JRL|8jqraot_xYJ<pF#KkBbm@Xgr|D8~f>1>nL)tiP@
zQ|O@FrL+t3{~J66k|eyB;i;Nuh5YP7Iae+e3%Qxu`ATj<Rjaw8T2`y`mAQO=dX_E@
z<Nv7|QLF!-w*UWbIDD1y@?_i+!yt<cQM3X%frTqvrfjXgNg6L*ZCHGuJVD4WRjf_M
zm$p8nV&WO349j2QP$zY<Px34w)A!g#CyyO)GAbnB0U$dX{NB?Cwm;Nf#7LbuMhbhG
z^SGBmuoTa;=_!9oVQywhnV-%r%oe8($P3Xs?WuRMJ*W~OQknD)Py|cy@Ta!YqJx5W
zG?b|Uq1zvgYU4;%0*21%<>mi-kQ#fynE?3dB?6zkc&SOE`pu8h&s}UIQOz&TB4a1=
zeZyCSutkGtJY+;oT$?#hU*cKvDY2_zHcJNhR$THjDtd+6WRW@$sRIw#A`V&7EZBBV
zYTG$Y;$d>x=cm94gdZgvhVbJ)Irh}F$S2puFWs?UL+-%g&K(Fbc%H=I@f5%lBoCy$
z?mzMUF6d%sHgn1#=wgTiKpcSVXpZ2uBMv};r+MH-90nl<5eI-c0INblJ7G4p6Of2u
zUx{!@{m_NS+B2X1%War-n`(%zQQ<zaR{9t4n5tDC^I#gfU{ll9*3W1ss{66~n#wDO
zlI_zpDDUNl_Kb$>e5Ax1ghWFeK%aQCUF<DUvLWICqDg5*F}uk@#I=4=bELamn2@#1
zPm<|qHAccmW3&N(n3XM7<uTVJYtCdIKmlDy0ZYpNpO4}H|EHg9zKl2kP9aK=K^Y{r
z4yPKDeT#DENrF|1-E*l&SCCUeb~LDKrw?p@SZjy_fOTdhtuslTB`BJLqA4hv(iUWG
zj!Hab2lD~SR6;d%>h*k9>SE^@5W4-*s75Rfz~leFk{WxZm(2I!y4NSSGff`<Zz<`E
zms)xJ(^HH2Jd&XXsUj%r8D}IDDUXcx+njt%RyAFF%+`!2p>5w(>WrKE`;aUGz83ge
zlxPLmLHJsdb6UMxumPnNeE!xnFI`lRs2%6bvsu^8c&EeA1r;KpaFaCPJFLmYO?iBu
z+~Iy|TK0m<zBa9ctxj*-Q7YS%V~O()HZ7xJ(#1<OZ)%YQqI9z+Ki|(l8YXBms#ST9
zuBe?u83<ngfmWI>La<~1b>=0ZzsZ6)bhsZnad~g>`p=^{Olq?B6v7km`%PWh3DUwU
zWs$Ux4uuYrGVRj;gYVSA{|Ep7A!!Vay2dj)9-IF^Pxs(Ee`4Sv9PjXOJleY?tDrgv
zz7+WXO@XO2uw0MM?&XC1|3Da0#$5=*Heo0@j3vr#FX&^A;V8N%3MZxVj^yR=I)Shw
zmD*D3lqFeYF&2!@Ra%y48_oHhL1dF)i^q~cVUIy&lh9EyMD~~y^Z#dk1&>(7kZEl)
zEhhV5hjDt>FyDM2>xQx2FdxWSFAX*wK^*|p0pJT$jhR|yn^6jnU3$~_=Vu0$l@hfT
zgJtVv{Qrd*{{Mf2|4*PhAHGs5f&VY^$Dj^Cp900=fd&7+FNv%VqO3)i=F{N<_J^$o
z{(orBhIM9?e9K9lDZXd9=<i@=u!G@zf-qxT?7}pA3XETbP@)b%naY*T6?5}#$@r3%
ztvDc#vV$RUI%|G$G=eDygl>N{DxaDn-S8_o4mn4~<Ns$<W7*;2?DykoK3N%$!~uLc
zeJRr#2Qag+n4d-*0O9~pdznh#MiOU%1B<-=+yUEzf@NJOuu*%?nP<Nu4gj{AQL@z}
zkA*@U0OA1lK*IqIXBcTXNln*%ivyrN@_+GtvSu$EdYG}yVNV@0dMKz$b~dY$4I-Ku
zAps}}5E5q*5`d5ZYb2A-AtV4L0Z<aaOTteoVpY@ibyKNm&JHEvCpC4`QdEuA9Y5%_
z&dIA;VmGVoCQTe}5cU$#g<8E6_zF-GfD$%nWHToeo2eZ*Vt(ESrED8|!{K<^4x?hR
zJabZotL<c0-Lx31)0xML-XOYgovJAC-E(l-Vrui`LVhn2>xqznJ~2IyYNtj>0F?n~
z27UEC>TAUF{}*HU|NrY{mT}@z1fV1ULIMyHuw(KdoD-CP>LHaa>gfcFt%q4`PaoL+
z5ZDkBfRKQZw9X{;s~{xcKqoUo0(xxoj|H@Q?B9S`)hG#&Gzys8w?d27Veu>o2}ro1
zN`!8IG^$|^2@w4M)bFPFpL6;1z5L}SDgU=$O=reCNcjaM=f4*i+R470C^<jbkafkh
z$$}hI<=wP2^^QRX=v~>Xy-P{`3lvZJR2Z+x6A+{@r(6?Zmj7tpB`@t0S)~2?n--M?
z0NG0h*(;o4LzHtVIrr70drkR)%BTZ?IshjyJfao~Ei=K|2PgAjoXp8(pl_N9gdmj?
zLcY03wuUaw<qnI%nU93>cUDbZFj|Wp{C}u~!>AG_wNrZvVUW3Ix_7dW_WS=sfB#QZ
z2Cxl_Cw~{5EnYsxQLIhIOV#wDLR(UIX!k@kJ#pF{X0w~kOF3zGS7ct?+mpoqlApa;
zoH0x~yt_+*6PB{KN7AfY`r$r^w4_*eH#qkM%S7h7QL!}-0NK&VJahWM_Q%2upFMo`
z@Y!=^gL9KxEzCS+G11Fu4@eef`0U}cmsQ14Y@^Y$NQ~a8P#rt)Q)1}#MnS#a7dK2|
zbh~S)gMxQ7l&Jxs+aHa}w=9iH8oy)5>vNw!%uK^*W<tz<kNJCDtnRc2MH#=?1io-S
z!udGxBIJnKN6dcDc8ZW_kZ1#!Xo%VGQz&M41yA*p!IuQ10x|oD*{=)I@~PD9>xkKJ
zCAd$`F6QT256$KmXJ&u&qiUWN^0N!&T)9vz<Ys2)E4c+#tx~m(vRa+5%;odbvmXEd
zJE;`?8J3^#y>ulK5-^q?zZ?h8uB_DSRC<_a`4en*U9*V+L3D?5L(`pAO2$Iu+gMvO
z<+-UUb%h-L4Nj()b2;fP?~s(saX${Li)0FZ{bA21?pO^*<ok+I7ugo+Yu092xt7mL
zg{dj}XO>tFQ<HShx>Doe=1a}qNe>$Nd|^&n(wRk+3|fP|h;&u4x5|cMsZ`jQx2kGh
z5(mHCs%BRd{e7lb8^q?ItUPsx82dygq5hj&2nmo?R#h50C%J^la>9f=rlK>)VY!N~
z*tS;TqgrVY=|iM3;>j_WDbW$5)H)w9%Lwg9;slau()^vJL`5|F*bRECEUnx?GEKQr
z*Eq#0IUN=u0SD!III7y#wZN+&bY&&O^eZxRA$$lYtS4-+ha|`~huC+EGD-rVBmhbR
p9D(_PkO0fiu6U%ls1BblLIT>V48o)blit8hdV>iGp#A<2{~uciMJoUR

diff --git a/etc/grafana.ini b/etc/grafana/grafana.ini
similarity index 100%
rename from etc/grafana.ini
rename to etc/grafana/grafana.ini
diff --git a/etc/grafana/provisioning/dashboards/foyer.yml b/etc/grafana/provisioning/dashboards/foyer.yml
new file mode 100644
index 00000000..8c23037a
--- /dev/null
+++ b/etc/grafana/provisioning/dashboards/foyer.yml
@@ -0,0 +1,9 @@
+apiVersion: 1
+
+providers:
+  - name: "foyer provider"
+    disableDeletion: true
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards/foyer.json
+      foldersFromFilesStructure: true
\ No newline at end of file
diff --git a/etc/grafana/provisioning/datasources/foyer.yml b/etc/grafana/provisioning/datasources/foyer.yml
new file mode 100644
index 00000000..187f4855
--- /dev/null
+++ b/etc/grafana/provisioning/datasources/foyer.yml
@@ -0,0 +1,14 @@
+apiVersion: 1
+deleteDatasources:
+  - name: foyer
+datasources:
+  - name: foyer
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    withCredentials: false
+    tlsAuth: false
+    tlsAuthWithCACert: false
+    version: 1
+    editable: true
+    isDefault: true
\ No newline at end of file
diff --git a/etc/prometheus.yml b/etc/prometheus/prometheus.yml
similarity index 100%
rename from etc/prometheus.yml
rename to etc/prometheus/prometheus.yml
diff --git a/scripts/monitor.sh b/scripts/monitor.sh
index 033d7447..23b9c9de 100755
--- a/scripts/monitor.sh
+++ b/scripts/monitor.sh
@@ -6,8 +6,6 @@ cat <<EOF > docker-compose.override.yaml
 version: '3'
 
 services:
-    grafana:
-        user: "${UID}"
     prometheus:
         user: "${UID}"
 EOF

From eba3317b054c1f51c5c1c11801e6cdeaa5fb0c3a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 10 Sep 2023 11:01:55 +0800
Subject: [PATCH 101/261] chore: bump toolchain (#130)

* chore: bump toolchain

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix build on mac

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix build

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix hakari

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .config/hakari.toml                | 5 +++++
 .github/template/template.yml      | 4 ++--
 .github/workflows/main.yml         | 4 ++--
 .github/workflows/pull-request.yml | 4 ++--
 Makefile                           | 4 ++--
 foyer-intrusive/src/lib.rs         | 7 ++-----
 foyer-storage-bench/src/utils.rs   | 4 ++--
 foyer-storage/src/device/fs.rs     | 1 +
 foyer-storage/src/store.rs         | 2 +-
 rust-toolchain                     | 2 +-
 10 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/.config/hakari.toml b/.config/hakari.toml
index 3c115dc3..f3329bba 100644
--- a/.config/hakari.toml
+++ b/.config/hakari.toml
@@ -22,3 +22,8 @@ platforms = [
 
 # Write out exact versions rather than a semver range. (Defaults to false.)
 # exact-versions = true
+
+[traversal-excludes]
+third-party = [
+    { name = "miniz_oxide" },
+]
\ No newline at end of file
diff --git a/.github/template/template.yml b/.github/template/template.yml
index eefdb765..f072d062 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -3,9 +3,9 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-31
+  RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230829
+  CACHE_KEY_SUFFIX: 20230910
 
 jobs:
   misc-check:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 342081c7..687ab8d2 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,9 +11,9 @@ on:
     branches: [main]
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-31
+  RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230829
+  CACHE_KEY_SUFFIX: 20230910
 jobs:
   misc-check:
     name: misc check
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index c81280fc..7ebf3567 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -10,9 +10,9 @@ on:
   pull_request:
     branches: [main]
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-31
+  RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230829
+  CACHE_KEY_SUFFIX: 20230910
 jobs:
   misc-check:
     name: misc check
diff --git a/Makefile b/Makefile
index a249a1bc..0b32a147 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: proto check test deps monitor
+.PHONY: proto check test deps monitor clear
 
 deps:
 	./scripts/install-deps.sh
@@ -23,4 +23,4 @@ monitor:
 	./scripts/monitor.sh
 
 clear:
-	rm -rf .tmp
\ No newline at end of file
+	rm -rf .tmp
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 09b0b191..ce6b0d32 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -44,11 +44,8 @@ pub use memoffset::offset_of;
 #[macro_export]
 macro_rules! container_of {
     ($ptr:expr, $container:path, $field:ident) => {
-        #[allow(clippy::cast_ptr_alignment)]
-        {
-            ($ptr as *const _ as *const u8).sub($crate::offset_of!($container, $field))
-                as *const $container
-        }
+        ($ptr as *const _ as *const u8).sub($crate::offset_of!($container, $field))
+            as *const $container
     };
 }
 
diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
index 630101c0..5c123a5d 100644
--- a/foyer-storage-bench/src/utils.rs
+++ b/foyer-storage-bench/src/utils.rs
@@ -31,7 +31,7 @@ use std::path::{Path, PathBuf};
 use itertools::Itertools;
 use nix::{fcntl::readlink, sys::stat::stat};
 
-#[allow(dead_code)]
+#[cfg_attr(not(target_os = "linux"), expect(dead_code))]
 #[derive(PartialEq, Clone, Copy, Debug)]
 pub enum FsType {
     Xfs,
@@ -41,7 +41,7 @@ pub enum FsType {
     Others,
 }
 
-#[allow(unused_variables)]
+#[cfg_attr(not(target_os = "linux"), expect(unused_variables))]
 pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
     #[cfg(target_os = "linux")]
     {
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 3c7ead57..184a0643 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -61,6 +61,7 @@ impl FsDeviceConfig {
 struct FsDeviceInner {
     config: FsDeviceConfig,
 
+    #[cfg_attr(not(target_os = "linux"), expect(dead_code))]
     dir: File,
 
     files: Vec<File>,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 464dba34..59b1ef60 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -920,7 +920,7 @@ where
             .unwrap();
 
         let Some(header) = EntryHeader::read(slice.as_ref()) else {
-            return Ok(None)
+            return Ok(None);
         };
 
         let entry_len = bits::align_up(
diff --git a/rust-toolchain b/rust-toolchain
index b0adca9d..20d4da12 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-05-31"
\ No newline at end of file
+channel = "nightly-2023-09-09"
\ No newline at end of file

From d7d52d7d7e74131fbc43a44c735c5842efd106c8 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 10 Sep 2023 11:18:38 +0800
Subject: [PATCH 102/261] chore: udeps (#131)

* chore: udeps

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update pr template

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/pull_request_template.md |  6 ------
 .github/template/template.yml    |  2 --
 foyer-intrusive/Cargo.toml       |  1 -
 foyer-memory/Cargo.toml          |  8 --------
 foyer-storage-bench/Cargo.toml   |  2 --
 foyer-storage/Cargo.toml         |  1 -
 foyer-workspace-hack/Cargo.toml  |  4 ++--
 foyer/Cargo.toml                 | 31 -------------------------------
 8 files changed, 2 insertions(+), 53 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index e33609bd..30795dd1 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -6,12 +6,6 @@
 
 Please explain **IN DETAIL** what the changes are in this PR and why they are needed:
 
-- Summarize your change (**mandatory**)
-- How does this PR work? Need a brief introduction for the changed logic (optional)
-- Describe clearly one logical change and avoid lazy messages (optional)
-- Describe any limitations of the current code (optional)
-- Refer to a related PR or issue link (optional)
-
 -->
 
 ## Checklist
diff --git a/.github/template/template.yml b/.github/template/template.yml
index f072d062..68521c2b 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -93,7 +93,6 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          # components: rustfmt, clippy, llvm-tools-preview
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
@@ -124,7 +123,6 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
-          # components: rustfmt, clippy, llvm-tools-preview
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 13720df3..2f9ea412 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -17,6 +17,5 @@ itertools = "0.10.5"
 memoffset = "0.9"
 parking_lot = "0.12"
 paste = "1.0"
-thiserror = "1"
 tracing = "0.1"
 twox-hash = "1"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index cb8ed171..0a39bfb0 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -8,25 +8,17 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-channel = "1.8"
-async-trait = "0.1"
-bitflags = "2.3.1"
 bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
-futures = "0.3"
 itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
-nix = { version = "0.27", features = ["fs", "mman"] }
 parking_lot = "0.12"
 paste = "1.0"
-pin-project = "1"
-prometheus = "0.13"
 rand = "0.8.5"
-thiserror = "1"
 tokio = { version = "1", features = [
     "rt",
     "rt-multi-thread",
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index ac51628f..f04d5160 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -27,8 +27,6 @@ opentelemetry-semantic-conventions = { version = "0.12", optional = true }
 parking_lot = "0.12"
 prometheus = "0.13"
 rand = "0.8.5"
-rand_mt = "4.2.1"
-tempfile = "3"
 tokio = { version = "1", features = [
     "rt",
     "rt-multi-thread",
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 690b5f7c..f88ce906 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -25,7 +25,6 @@ memoffset = "0.9"
 nix = { version = "0.27", features = ["fs", "mman", "uio"] }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
-pin-project = "1"
 prometheus = "0.13"
 rand = "0.8.5"
 thiserror = "1"
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 0eaf1126..1e87c402 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -13,7 +13,6 @@ publish = false
 
 ### BEGIN HAKARI SECTION
 [dependencies]
-crossbeam-channel = { version = "0.5" }
 crossbeam-utils = { version = "0.8" }
 either = { version = "1", default-features = false, features = ["use_std"] }
 futures-channel = { version = "0.3", features = ["sink"] }
@@ -22,8 +21,9 @@ futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
 hyper = { version = "0.14", features = ["full"] }
 itertools = { version = "0.10" }
+libc = { version = "0.2", features = ["extra_traits"] }
 lock_api = { version = "0.4", features = ["arc_lock"] }
-nix = { version = "0.27", features = ["fs", "mman", "uio"] }
+memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 0c1352d8..901bafcc 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -8,38 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1"
-bytes = "1"
-cmsketch = "0.1"
-crossbeam = "0.8"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-storage = { path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
-futures = "0.3"
-itertools = "0.11.0"
-libc = "0.2"
-memoffset = "0.9"
-nix = { version = "0.27", features = ["fs", "mman"] }
-parking_lot = "0.12"
-paste = "1.0"
-prometheus = "0.13"
-rand = "0.8.5"
-thiserror = "1"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
-tracing = "0.1"
-twox-hash = "1"
-
-[dev-dependencies]
-bytesize = "1"
-clap = { version = "4", features = ["derive"] }
-hdrhistogram = "7"
-rand_mt = "4.2.1"
-tempfile = "3"

From 93fb2e6af16dbf7ccab02e06eb7e652f4562dc7c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Sep 2023 14:08:22 +0800
Subject: [PATCH 103/261] chore: use uint gauge instead (#132)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/flusher.rs   |  2 +-
 foyer-storage/src/metrics.rs   | 33 +++++++++++++++++++++++++++------
 foyer-storage/src/reclaimer.rs |  2 +-
 foyer-storage/src/store.rs     |  3 +++
 4 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 2f98cc90..72ebf381 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -137,7 +137,7 @@ where
             .inc_by(region.device().region_size() as u64);
         self.metrics
             .total_bytes
-            .add(region.device().region_size() as i64);
+            .add(region.device().region_size() as u64);
 
         Ok(())
     }
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index be6007b7..01decf3c 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -15,10 +15,31 @@
 use std::sync::{LazyLock, OnceLock};
 
 use prometheus::{
-    register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
-    register_int_gauge_vec_with_registry, Histogram, HistogramVec, IntCounter, IntCounterVec,
-    IntGauge, IntGaugeVec, Registry,
+    core::{AtomicU64, GenericGauge, GenericGaugeVec},
+    opts, register_histogram_vec_with_registry, register_int_counter_vec_with_registry, Histogram,
+    HistogramVec, IntCounter, IntCounterVec, Registry,
 };
+type UintGaugeVec = GenericGaugeVec<AtomicU64>;
+type UintGauge = GenericGauge<AtomicU64>;
+
+macro_rules! register_gauge_vec {
+    ($TYPE:ident, $OPTS:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
+        let gauge_vec = $TYPE::new($OPTS, $LABELS_NAMES).unwrap();
+        $REGISTRY
+            .register(Box::new(gauge_vec.clone()))
+            .map(|_| gauge_vec)
+    }};
+}
+
+macro_rules! register_uint_gauge_vec_with_registry {
+    ($OPTS:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
+        register_gauge_vec!(UintGaugeVec, $OPTS, $LABELS_NAMES, $REGISTRY)
+    }};
+
+    ($NAME:expr, $HELP:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
+        register_uint_gauge_vec_with_registry!(opts!($NAME, $HELP), $LABELS_NAMES, $REGISTRY)
+    }};
+}
 
 static REGISTRY: OnceLock<Registry> = OnceLock::new();
 
@@ -43,7 +64,7 @@ pub struct GlobalMetrics {
     op_duration: HistogramVec,
     slow_op_duration: HistogramVec,
     op_bytes: IntCounterVec,
-    total_bytes: IntGaugeVec,
+    total_bytes: UintGaugeVec,
 
     inner_op_duration: HistogramVec,
 }
@@ -82,7 +103,7 @@ impl GlobalMetrics {
         )
         .unwrap();
 
-        let total_bytes = register_int_gauge_vec_with_registry!(
+        let total_bytes = register_uint_gauge_vec_with_registry!(
             "foyer_storage_total_bytes",
             "foyer storage total bytes",
             &["foyer"],
@@ -131,7 +152,7 @@ pub struct Metrics {
     pub op_bytes_reclaim: IntCounter,
     pub op_bytes_reinsert: IntCounter,
 
-    pub total_bytes: IntGauge,
+    pub total_bytes: UintGauge,
 
     pub inner_op_duration_acquire_clean_region: Histogram,
     pub inner_op_duration_acquire_clean_buffer: Histogram,
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index ad882a92..e391cf8b 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -217,7 +217,7 @@ where
             .inc_by(region.device().region_size() as u64);
         self.metrics
             .total_bytes
-            .sub(region.device().region_size() as i64);
+            .sub(region.device().region_size() as u64);
 
         Ok(())
     }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 59b1ef60..385e1ee3 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -530,6 +530,9 @@ where
         }
 
         tracing::info!("finish store recovery, {} region recovered", recovered);
+        self.metrics
+            .total_bytes
+            .set((recovered * self.device.region_size()) as u64);
 
         // Force trigger reclamation.
         if recovered == self.device.regions() {

From 84a0554199257afce8f37047c3a0c967da029ff6 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Sep 2023 17:41:46 +0800
Subject: [PATCH 104/261] fix: panic on recovery if pread len not match (#133)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region.rs | 7 +++++++
 foyer-storage/src/store.rs  | 8 ++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 0a6fc92f..d155b54c 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -187,6 +187,13 @@ where
         }
     }
 
+    /// Load region data into a [`ReadSlice`].
+    ///
+    /// Data may be loaded ether from physical device or from dirty buffer.
+    ///
+    /// Use version `0` to skip version check.
+    ///
+    /// Returns `None` if verion mismatch or given range cannot be fully filled.
     #[tracing::instrument(skip(self, range), fields(start, end))]
     pub async fn load(
         &self,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 385e1ee3..a4b431e2 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -953,7 +953,9 @@ where
             key
         } else {
             drop(slice);
-            let s = self.region.load(align_start..align_end, 0).await?.unwrap();
+            let Some(s) = self.region.load(align_start..align_end, 0).await? else {
+                return Ok(None);
+            };
             let rel_start = abs_start - align_start;
             let rel_end = abs_end - align_start;
 
@@ -987,7 +989,9 @@ where
         // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
         let start = index.offset as usize;
         let end = start + index.len as usize;
-        let slice = self.region.load(start..end, 0).await?.unwrap();
+        let Some(slice) = self.region.load(start..end, 0).await? else {
+            return Ok(None);
+        };
         let kv = read_entry::<K, V>(slice.as_ref());
         drop(slice);
 

From 2c6f080835f49fc6026b97fd53d389bae81e9361 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Sep 2023 17:47:53 +0800
Subject: [PATCH 105/261] chore: simplify pr template (#134)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/pull_request_template.md | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 30795dd1..5875db12 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,12 +1,6 @@
 ## What's changed and what's your intention?
 
-<!--
-
-**Please do not leave this empty!**
-
-Please explain **IN DETAIL** what the changes are in this PR and why they are needed:
-
--->
+> Please explain **IN DETAIL** what the changes are in this PR and why they are needed. :D
 
 ## Checklist
 

From 4e81665c0c5f1821dd3f2e170ab0db3e18b61c23 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 13 Sep 2023 13:10:26 +0800
Subject: [PATCH 106/261] feat: introduce rated ticket admission & reinsertion
 policy (#135)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml                 |   3 +-
 .github/workflows/main.yml                    |   3 +-
 .github/workflows/pull-request.yml            |   3 +-
 Makefile                                      |  11 +-
 foyer-common/src/lib.rs                       |   1 +
 foyer-common/src/rated_random.rs              |  36 +---
 foyer-common/src/rated_ticket.rs              | 182 ++++++++++++++++++
 foyer-storage-bench/src/main.rs               |  30 ++-
 foyer-storage/src/admission/mod.rs            |   1 +
 foyer-storage/src/admission/rated_random.rs   |  19 +-
 foyer-storage/src/admission/rated_ticket.rs   |  68 +++++++
 foyer-storage/src/reinsertion/mod.rs          |   1 +
 foyer-storage/src/reinsertion/rated_random.rs |  19 +-
 foyer-storage/src/reinsertion/rated_ticket.rs |  68 +++++++
 14 files changed, 392 insertions(+), 53 deletions(-)
 create mode 100644 foyer-common/src/rated_ticket.rs
 create mode 100644 foyer-storage/src/admission/rated_ticket.rs
 create mode 100644 foyer-storage/src/reinsertion/rated_ticket.rs

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 68521c2b..cdf5a6f7 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -76,8 +76,7 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-common --lib -- rated_random::tests --nocapture --ignored
-          cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common -- --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov --no-report nextest
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 687ab8d2..f75aef45 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -83,8 +83,7 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-common --lib -- rated_random::tests --nocapture --ignored
-          cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common -- --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov --no-report nextest
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 7ebf3567..b28bc1f6 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -82,8 +82,7 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-common --lib -- rated_random::tests --nocapture --ignored
-          cargo llvm-cov --no-report test --package foyer-common --lib -- rate::tests --nocapture --ignored
+          cargo llvm-cov --no-report test --package foyer-common -- --nocapture --ignored
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov --no-report nextest
diff --git a/Makefile b/Makefile
index 0b32a147..f9238be3 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: proto check test deps monitor clear
+.PHONY: deps check test test-ignored test-all all monitor clear
 
 deps:
 	./scripts/install-deps.sh
@@ -18,7 +18,14 @@ check:
 test:
 	RUST_BACKTRACE=1 cargo nextest run --all
 	RUST_BACKTRACE=1 cargo test --doc
-	
+
+test-ignored:
+	RUST_BACKTRACE=1 cargo test --package foyer-common -- --nocapture --ignored
+
+test-all: test test-ignored
+
+all: check test-all
+
 monitor:
 	./scripts/monitor.sh
 
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 8b8e5e1a..d3208b94 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -22,3 +22,4 @@ pub mod code;
 pub mod queue;
 pub mod rate;
 pub mod rated_random;
+pub mod rated_ticket;
diff --git a/foyer-common/src/rated_random.rs b/foyer-common/src/rated_random.rs
index dfd76811..5ce2460f 100644
--- a/foyer-common/src/rated_random.rs
+++ b/foyer-common/src/rated_random.rs
@@ -14,12 +14,10 @@
 
 use std::{
     fmt::Debug,
-    marker::PhantomData,
     sync::atomic::{AtomicUsize, Ordering},
     time::{Duration, Instant},
 };
 
-use crate::code::{Key, Value};
 use parking_lot::{Mutex, MutexGuard};
 use rand::{thread_rng, Rng};
 
@@ -51,11 +49,7 @@ const PRECISION: usize = 100000;
 ///
 /// p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
 #[derive(Debug)]
-pub struct RatedRandom<K, V>
-where
-    K: Key,
-    V: Value,
-{
+pub struct RatedRandom {
     rate: usize,
     update_interval: Duration,
 
@@ -65,8 +59,6 @@ where
     probability: AtomicUsize,
 
     inner: Mutex<Inner>,
-
-    _marker: PhantomData<(K, V)>,
 }
 
 #[derive(Debug)]
@@ -76,11 +68,7 @@ struct Inner {
     last_force_insert_bytes: usize,
 }
 
-impl<K, V> RatedRandom<K, V>
-where
-    K: Key,
-    V: Value,
-{
+impl RatedRandom {
     pub fn new(rate: usize, update_interval: Duration) -> Self {
         Self {
             rate,
@@ -95,11 +83,10 @@ where
                 last_obey_bytes: 0,
                 last_force_insert_bytes: 0,
             }),
-            _marker: PhantomData,
         }
     }
 
-    pub fn judge(&self, _key: &K, _weight: usize) -> bool {
+    pub fn judge(&self) -> bool {
         if let Some(inner) = self.inner.try_lock() {
             self.update(inner);
         }
@@ -107,7 +94,7 @@ where
         thread_rng().gen_range(0..PRECISION) < self.probability.load(Ordering::Relaxed)
     }
 
-    pub fn on_insert(&self, _key: &K, weight: usize, judge: bool) {
+    pub fn on_insert(&self, weight: usize, judge: bool) {
         if judge {
             // obey
             self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
@@ -117,7 +104,7 @@ where
         }
     }
 
-    pub fn on_drop(&self, _key: &K, weight: usize, judge: bool) {
+    pub fn on_drop(&self, weight: usize, judge: bool) {
         if !judge {
             // obey
             self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
@@ -204,21 +191,18 @@ mod tests {
 
         let score = Arc::new(AtomicUsize::new(0));
 
-        let rr = Arc::new(RatedRandom::<u64, Vec<u8>>::new(
-            RATE,
-            Duration::from_millis(100),
-        ));
+        let rr = Arc::new(RatedRandom::new(RATE, Duration::from_millis(100)));
 
         // scope: CONCURRENCY * (1 / interval) * range
         // [1_000_000, 10_000_000]
         // FORCE: [100_000, 1_000_000]
 
-        async fn submit(rr: Arc<RatedRandom<u64, Vec<u8>>>, score: Arc<AtomicUsize>) {
+        async fn submit(rr: Arc<RatedRandom>, score: Arc<AtomicUsize>) {
             loop {
                 tokio::time::sleep(Duration::from_millis(1)).await;
                 let weight = thread_rng().gen_range(100..1000);
 
-                let judge = rr.judge(&0, weight);
+                let judge = rr.judge();
                 let p_other = thread_rng().gen_range(0.0..=1.0);
                 let p_force = thread_rng().gen_range(0.0..=1.0);
 
@@ -226,9 +210,9 @@ mod tests {
 
                 if insert {
                     score.fetch_add(weight, Ordering::Relaxed);
-                    rr.on_insert(&0, weight, judge);
+                    rr.on_insert(weight, judge);
                 } else {
-                    rr.on_drop(&0, weight, judge);
+                    rr.on_drop(weight, judge);
                 }
             }
         }
diff --git a/foyer-common/src/rated_ticket.rs b/foyer-common/src/rated_ticket.rs
new file mode 100644
index 00000000..3cf94707
--- /dev/null
+++ b/foyer-common/src/rated_ticket.rs
@@ -0,0 +1,182 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::time::Instant;
+
+use parking_lot::Mutex;
+
+#[derive(Debug)]
+pub struct RatedTicket {
+    inner: Mutex<Inner>,
+    rate: f64,
+}
+
+#[derive(Debug)]
+struct Inner {
+    quota: f64,
+
+    last: Instant,
+}
+
+impl RatedTicket {
+    pub fn new(rate: f64) -> Self {
+        let inner = Inner {
+            quota: 0.0,
+            last: Instant::now(),
+        };
+        Self {
+            rate,
+            inner: Mutex::new(inner),
+        }
+    }
+
+    pub fn probe(&self) -> bool {
+        let mut inner = self.inner.lock();
+
+        let now = Instant::now();
+        let refill = now.duration_since(inner.last).as_secs_f64() * self.rate;
+        inner.last = now;
+        inner.quota = f64::min(inner.quota + refill, self.rate);
+
+        inner.quota > 0.0
+    }
+
+    pub fn reduce(&self, weight: f64) {
+        self.inner.lock().quota -= weight;
+    }
+
+    pub fn consume(&self, weight: f64) -> bool {
+        let mut inner = self.inner.lock();
+
+        let now = Instant::now();
+        let refill = now.duration_since(inner.last).as_secs_f64() * self.rate;
+        inner.last = now;
+        inner.quota = f64::min(inner.quota + refill, self.rate);
+
+        if inner.quota <= 0.0 {
+            return false;
+        }
+
+        inner.quota -= weight;
+
+        true
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        sync::{
+            atomic::{AtomicUsize, Ordering},
+            Arc,
+        },
+        time::Duration,
+    };
+
+    use itertools::Itertools;
+    use rand::{thread_rng, Rng};
+
+    use super::*;
+
+    #[ignore]
+    #[test]
+    fn test_rated_ticket_consume() {
+        test(consume)
+    }
+
+    #[ignore]
+    #[test]
+    fn test_rated_ticket_probe_reduce() {
+        test(probe_reduce)
+    }
+
+    fn test<F>(f: F)
+    where
+        F: Fn(usize, &Arc<AtomicUsize>, &Arc<RatedTicket>) + Send + Sync + Copy + 'static,
+    {
+        const CASES: usize = 10;
+        const ERATIO: f64 = 0.05;
+
+        let handles = (0..CASES)
+            .map(|_| std::thread::spawn(move || case(f)))
+            .collect_vec();
+        let mut eratios = vec![];
+        for handle in handles {
+            let eratio = handle.join().unwrap();
+            assert!(eratio < ERATIO, "eratio: {} < ERATIO: {}", eratio, ERATIO);
+            eratios.push(eratio);
+        }
+        println!("========== RatedTicket error ratio begin ==========");
+        for eratio in eratios {
+            println!("eratio: {eratio}");
+        }
+        println!("=========== RatedTicket error ratio end ===========");
+    }
+
+    fn consume(weight: usize, v: &Arc<AtomicUsize>, limiter: &Arc<RatedTicket>) {
+        if limiter.consume(weight as f64) {
+            v.fetch_add(weight, Ordering::Relaxed);
+        }
+    }
+
+    fn probe_reduce(weight: usize, v: &Arc<AtomicUsize>, limiter: &Arc<RatedTicket>) {
+        if limiter.probe() {
+            limiter.reduce(weight as f64);
+            v.fetch_add(weight, Ordering::Relaxed);
+        }
+    }
+
+    fn case<F>(f: F) -> f64
+    where
+        F: Fn(usize, &Arc<AtomicUsize>, &Arc<RatedTicket>) + Send + Sync + Copy + 'static,
+    {
+        const THREADS: usize = 8;
+        const RATE: usize = 1000;
+        const DURATION: Duration = Duration::from_secs(10);
+
+        let v = Arc::new(AtomicUsize::new(0));
+        let limiter = Arc::new(RatedTicket::new(RATE as f64));
+        let task = |rate: usize, v: Arc<AtomicUsize>, limiter: Arc<RatedTicket>, f: F| {
+            let start = Instant::now();
+            let mut rng = thread_rng();
+            loop {
+                if start.elapsed() >= DURATION {
+                    break;
+                }
+                std::thread::sleep(Duration::from_millis(rng.gen_range(1..100)));
+                f(rate, &v, &limiter)
+            }
+        };
+        let mut handles = vec![];
+        let mut rng = thread_rng();
+        for _ in 0..THREADS {
+            let rate = rng.gen_range(10..20);
+            let handle = std::thread::spawn({
+                let v = v.clone();
+                let limiter = limiter.clone();
+                move || task(rate, v, limiter, f)
+            });
+            handles.push(handle);
+        }
+
+        for handle in handles {
+            handle.join().unwrap();
+        }
+
+        let error = (v.load(Ordering::Relaxed) as isize
+            - RATE as isize * DURATION.as_secs() as isize)
+            .unsigned_abs();
+        error as f64 / (RATE as f64 * DURATION.as_secs_f64())
+    }
+}
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index a84a58c5..ca5e770b 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -35,9 +35,15 @@ use clap::Parser;
 
 use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
-    admission::{rated_random::RatedRandomAdmissionPolicy, AdmissionPolicy},
+    admission::{
+        rated_random::RatedRandomAdmissionPolicy, rated_ticket::RatedTicketAdmissionPolicy,
+        AdmissionPolicy,
+    },
     device::fs::FsDeviceConfig,
-    reinsertion::{rated_random::RatedRandomReinsertionPolicy, ReinsertionPolicy},
+    reinsertion::{
+        rated_random::RatedRandomReinsertionPolicy, rated_ticket::RatedTicketReinsertionPolicy,
+        ReinsertionPolicy,
+    },
     store::StoreConfig,
     LfuFsStore,
 };
@@ -125,11 +131,21 @@ pub struct Args {
     #[arg(long, default_value_t = 0)]
     rated_random_admission: usize,
 
-    /// enable rated random admission policy if `rated_random` > 0
+    /// enable rated random reinsertion policy if `rated_random` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     rated_random_reinsertion: usize,
 
+    /// enable rated ticket admission policy if `rated_random` > 0
+    /// (MiB/s)
+    #[arg(long, default_value_t = 0)]
+    rated_ticket_admission: usize,
+
+    /// enable rated ticket reinsetion policy if `rated_random` > 0
+    /// (MiB/s)
+    #[arg(long, default_value_t = 0)]
+    rated_ticket_reinsertion: usize,
+
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     flush_rate_limit: usize,
@@ -295,6 +311,14 @@ async fn main() {
         );
         reinsertions.push(Arc::new(rr));
     }
+    if args.rated_ticket_admission > 0 {
+        let rt = RatedTicketAdmissionPolicy::new(args.rated_ticket_admission * 1024 * 1024);
+        admissions.push(Arc::new(rt));
+    }
+    if args.rated_ticket_reinsertion > 0 {
+        let rt = RatedTicketReinsertionPolicy::new(args.rated_ticket_reinsertion * 1024 * 1024);
+        reinsertions.push(Arc::new(rt));
+    }
 
     let clean_region_threshold = if args.clean_region_threshold == 0 {
         args.reclaimers
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 5a806abe..9c5c8577 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -33,3 +33,4 @@ pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
 }
 
 pub mod rated_random;
+pub mod rated_ticket;
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
index 79675754..6a235ca2 100644
--- a/foyer-storage/src/admission/rated_random.rs
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, sync::Arc, time::Duration};
+use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Duration};
 
 use foyer_common::{
     code::{Key, Value},
@@ -29,7 +29,9 @@ where
     K: Key,
     V: Value,
 {
-    inner: RatedRandom<K, V>,
+    inner: RatedRandom,
+
+    _marker: PhantomData<(K, V)>,
 }
 
 impl<K, V> RatedRandomAdmissionPolicy<K, V>
@@ -40,6 +42,7 @@ where
     pub fn new(rate: usize, update_interval: Duration) -> Self {
         Self {
             inner: RatedRandom::new(rate, update_interval),
+            _marker: PhantomData,
         }
     }
 }
@@ -53,15 +56,15 @@ where
 
     type Value = V;
 
-    fn judge(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>) -> bool {
-        self.inner.judge(key, weight)
+    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.inner.judge()
     }
 
-    fn on_insert(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
-        self.inner.on_insert(key, weight, judge)
+    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_insert(weight, judge)
     }
 
-    fn on_drop(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
-        self.inner.on_drop(key, weight, judge)
+    fn on_drop(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_drop(weight, judge)
     }
 }
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
new file mode 100644
index 00000000..958c3c6d
--- /dev/null
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -0,0 +1,68 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, marker::PhantomData, sync::Arc};
+
+use foyer_common::{
+    code::{Key, Value},
+    rated_ticket::RatedTicket,
+};
+
+use crate::metrics::Metrics;
+
+use super::AdmissionPolicy;
+
+#[derive(Debug)]
+pub struct RatedTicketAdmissionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    inner: RatedTicket,
+
+    _marker: PhantomData<(K, V)>,
+}
+
+impl<K, V> RatedTicketAdmissionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn new(rate: usize) -> Self {
+        Self {
+            inner: RatedTicket::new(rate as f64),
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<K, V> AdmissionPolicy for RatedTicketAdmissionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.inner.probe()
+    }
+
+    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {
+        self.inner.reduce(weight as f64);
+    }
+
+    fn on_drop(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+}
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index 448bdc39..bf3881de 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -33,3 +33,4 @@ pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
 
 pub mod exist;
 pub mod rated_random;
+pub mod rated_ticket;
diff --git a/foyer-storage/src/reinsertion/rated_random.rs b/foyer-storage/src/reinsertion/rated_random.rs
index 7c570268..ab10038c 100644
--- a/foyer-storage/src/reinsertion/rated_random.rs
+++ b/foyer-storage/src/reinsertion/rated_random.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, sync::Arc, time::Duration};
+use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Duration};
 
 use foyer_common::{
     code::{Key, Value},
@@ -29,7 +29,9 @@ where
     K: Key,
     V: Value,
 {
-    inner: RatedRandom<K, V>,
+    inner: RatedRandom,
+
+    _marker: PhantomData<(K, V)>,
 }
 
 impl<K, V> RatedRandomReinsertionPolicy<K, V>
@@ -40,6 +42,7 @@ where
     pub fn new(rate: usize, update_interval: Duration) -> Self {
         Self {
             inner: RatedRandom::new(rate, update_interval),
+            _marker: PhantomData,
         }
     }
 }
@@ -53,15 +56,15 @@ where
 
     type Value = V;
 
-    fn judge(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>) -> bool {
-        self.inner.judge(key, weight)
+    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.inner.judge()
     }
 
-    fn on_insert(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
-        self.inner.on_insert(key, weight, judge)
+    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_insert(weight, judge)
     }
 
-    fn on_drop(&self, key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
-        self.inner.on_drop(key, weight, judge)
+    fn on_drop(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+        self.inner.on_drop(weight, judge)
     }
 }
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
new file mode 100644
index 00000000..281c01ba
--- /dev/null
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -0,0 +1,68 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, marker::PhantomData, sync::Arc};
+
+use foyer_common::{
+    code::{Key, Value},
+    rated_ticket::RatedTicket,
+};
+
+use crate::metrics::Metrics;
+
+use super::ReinsertionPolicy;
+
+#[derive(Debug)]
+pub struct RatedTicketReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    inner: RatedTicket,
+
+    _marker: PhantomData<(K, V)>,
+}
+
+impl<K, V> RatedTicketReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn new(rate: usize) -> Self {
+        Self {
+            inner: RatedTicket::new(rate as f64),
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<K, V> ReinsertionPolicy for RatedTicketReinsertionPolicy<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.inner.probe()
+    }
+
+    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {
+        self.inner.reduce(weight as f64);
+    }
+
+    fn on_drop(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+}

From 69926a041cfc6ca5803ea87da159a335322a63b0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 13 Sep 2023 14:10:12 +0800
Subject: [PATCH 107/261] fix: fix foyer storage bench tool analysis (#137)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/analyze.rs |  9 +++++++++
 foyer-storage-bench/src/main.rs    | 13 ++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index 02a9c234..211f9046 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -61,6 +61,7 @@ pub struct Analysis {
 
     get_iops: f64,
     get_miss: f64,
+    get_throughput: f64,
     get_miss_lat_p50: u64,
     get_miss_lat_p90: u64,
     get_miss_lat_p99: u64,
@@ -82,6 +83,7 @@ pub struct MetricsDump {
 
     pub get_ios: usize,
     pub get_miss_ios: usize,
+    pub get_bytes: usize,
     pub get_hit_lat_p50: u64,
     pub get_hit_lat_p90: u64,
     pub get_hit_lat_p99: u64,
@@ -99,6 +101,7 @@ pub struct Metrics {
     pub insert_lats: Arc<RwLock<Histogram<u64>>>,
 
     pub get_ios: Arc<AtomicUsize>,
+    pub get_bytes: Arc<AtomicUsize>,
     pub get_miss_ios: Arc<AtomicUsize>,
     pub get_hit_lats: Arc<RwLock<Histogram<u64>>>,
     pub get_miss_lats: Arc<RwLock<Histogram<u64>>>,
@@ -114,6 +117,7 @@ impl Default for Metrics {
             )),
 
             get_ios: Arc::new(AtomicUsize::new(0)),
+            get_bytes: Arc::new(AtomicUsize::new(0)),
             get_miss_ios: Arc::new(AtomicUsize::new(0)),
             get_hit_lats: Arc::new(RwLock::new(
                 Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
@@ -141,6 +145,7 @@ impl Metrics {
 
             get_ios: self.get_ios.load(Ordering::Relaxed),
             get_miss_ios: self.get_miss_ios.load(Ordering::Relaxed),
+            get_bytes: self.get_bytes.load(Ordering::Relaxed),
             get_hit_lat_p50: get_hit_lats.value_at_quantile(0.5),
             get_hit_lat_p90: get_hit_lats.value_at_quantile(0.9),
             get_hit_lat_p99: get_hit_lats.value_at_quantile(0.99),
@@ -197,8 +202,10 @@ impl std::fmt::Display for Analysis {
         writeln!(f, "insert lat pmax: {}us", self.insert_lat_pmax)?;
 
         // get statics
+        let get_throughput = ByteSize::b(self.get_throughput as u64);
         writeln!(f, "get iops: {:.1}/s", self.get_iops)?;
         writeln!(f, "get miss: {:.2}% ", self.get_miss * 100f64)?;
+        writeln!(f, "get throughput: {}/s", get_throughput.to_string_as(true))?;
         writeln!(f, "get hit lat p50: {}us", self.get_hit_lat_p50)?;
         writeln!(f, "get hit lat p90: {}us", self.get_hit_lat_p90)?;
         writeln!(f, "get hit lat p99: {}us", self.get_hit_lat_p99)?;
@@ -234,6 +241,7 @@ pub fn analyze(
     let get_iops = (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64 / secs;
     let get_miss = (metrics_dump_end.get_miss_ios - metrics_dump_start.get_miss_ios) as f64
         / (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64;
+    let get_throughput = (metrics_dump_end.get_bytes - metrics_dump_start.get_bytes) as f64 / secs;
 
     Analysis {
         disk_read_iops,
@@ -250,6 +258,7 @@ pub fn analyze(
 
         get_iops,
         get_miss,
+        get_throughput,
         get_hit_lat_p50: metrics_dump_end.get_hit_lat_p50,
         get_hit_lat_p90: metrics_dump_end.get_hit_lat_p90,
         get_hit_lat_p99: metrics_dump_end.get_hit_lat_p99,
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index ca5e770b..5206041c 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -469,16 +469,18 @@ async fn write(
         }
 
         let time = Instant::now();
-        store.insert(idx, data).await.unwrap();
+        let inserted = store.insert(idx, data).await.unwrap();
         let lat = time.elapsed().as_micros() as u64;
         if let Err(e) = metrics.insert_lats.write().record(lat) {
             tracing::error!("metrics error: {:?}, value: {}", e, lat);
         }
 
-        metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
-        metrics
-            .insert_bytes
-            .fetch_add(entry_size, Ordering::Relaxed);
+        if inserted {
+            metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
+            metrics
+                .insert_bytes
+                .fetch_add(entry_size, Ordering::Relaxed);
+        }
     }
 }
 
@@ -524,6 +526,7 @@ async fn read(
             if let Err(e) = metrics.get_hit_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);
             }
+            metrics.get_bytes.fetch_add(entry_size, Ordering::Relaxed);
         } else {
             if let Err(e) = metrics.get_miss_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);

From 2d886ba8f477447589b9a124d51189ab3f44380f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 13 Sep 2023 14:35:43 +0800
Subject: [PATCH 108/261] feat: introduce force insert interface (#138)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/judge.rs |  5 +++
 foyer-storage/src/store.rs | 76 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)

diff --git a/foyer-storage/src/judge.rs b/foyer-storage/src/judge.rs
index 069498ff..c3fa7bdd 100644
--- a/foyer-storage/src/judge.rs
+++ b/foyer-storage/src/judge.rs
@@ -57,6 +57,11 @@ impl Judges {
         self.judge = judge;
     }
 
+    pub fn set_mask(&mut self, mut mask: Bitmap<64>) {
+        mask.invert();
+        self.umask = mask;
+    }
+
     /// judge | ( ~mask )
     ///
     /// | judge | mask | ~mask | result |
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index a4b431e2..fa92beed 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -19,6 +19,7 @@ use std::{
     time::{Duration, Instant},
 };
 
+use bitmaps::Bitmap;
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, rate::RateLimiter};
 use foyer_intrusive::eviction::EvictionPolicy;
@@ -304,6 +305,16 @@ where
         self.insert(key, value).await
     }
 
+    #[tracing::instrument(skip(self, value))]
+    pub async fn insert_force(&self, key: K, value: V) -> Result<bool> {
+        let weight = key.serialized_len() + value.serialized_len();
+        let mut writer = StoreWriter::new(self, key, weight);
+        writer.set_force();
+        let inserted = writer.finish(value).await?;
+        assert!(inserted);
+        Ok(inserted)
+    }
+
     /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
     /// Then `f` will be called and entry will be inserted.
     ///
@@ -329,6 +340,34 @@ where
         writer.finish(value).await
     }
 
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    pub async fn insert_force_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
+    where
+        F: FnOnce() -> anyhow::Result<V>,
+    {
+        let mut writer = self.writer(key, weight);
+        writer.set_force();
+        if !writer.judge() {
+            return Ok(false);
+        }
+        let value = match f() {
+            Ok(value) => value,
+            Err(e) => {
+                tracing::warn!("fetch value error: {:?}", e);
+                return Ok(false);
+            }
+        };
+        let inserted = writer.finish(value).await?;
+        assert!(inserted);
+        Ok(inserted)
+    }
+
     /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
     /// Then `f` will be called to fetch value, and entry will be inserted.
     ///
@@ -355,6 +394,35 @@ where
         writer.finish(value).await
     }
 
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called to fetch value, and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    pub async fn insert_force_with_future<F, FU>(&self, key: K, f: F, weight: usize) -> Result<bool>
+    where
+        F: FnOnce() -> FU,
+        FU: FetchValueFuture<V>,
+    {
+        let mut writer = self.writer(key, weight);
+        writer.set_force();
+        if !writer.judge() {
+            return Ok(false);
+        }
+        let value = match f().await {
+            Ok(value) => value,
+            Err(e) => {
+                tracing::warn!("fetch value error: {:?}", e);
+                return Ok(false);
+            }
+        };
+        let inserted = writer.finish(value).await?;
+        assert!(inserted);
+        Ok(inserted)
+    }
+
     #[tracing::instrument(skip(self, f))]
     pub async fn insert_if_not_exists_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
     where
@@ -702,6 +770,14 @@ where
         self.store.apply_writer(self, value).await
     }
 
+    pub fn set_force(&mut self) {
+        self.judges.set_mask(Bitmap::new());
+    }
+
+    pub fn set_judge_mask(&mut self, mask: Bitmap<64>) {
+        self.judges.set_mask(mask);
+    }
+
     pub fn set_skippable(&mut self) {
         self.is_skippable = true
     }

From 41b1d3934cc92976737a9296273b4c5bee6422a0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 13 Sep 2023 14:36:24 +0800
Subject: [PATCH 109/261] chore: update pr template (#139)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/pull_request_template.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 5875db12..5d397a96 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -6,6 +6,6 @@
 
 - [ ] I have written necessary rustdoc comments
 - [ ] I have added necessary unit tests and integration tests
-- [ ] I have passed `make check` and `make test` in my local envirorment.
+- [ ] I have passed `make check` and `make test` or `make all` in my local envirorment.
 
 ## Related issues or PRs (optional)

From ef773f2bff63802486cae4360f7f67f51bdcc872 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 25 Sep 2023 15:19:10 +0800
Subject: [PATCH 110/261] refactor: refine allocation to make it fair (#140)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs     |   5 ++
 foyer-storage/src/reclaimer.rs      |  25 +++++--
 foyer-storage/src/region.rs         |   3 +-
 foyer-storage/src/region_manager.rs | 106 +++++++++++-----------------
 foyer-storage/src/store.rs          |   9 ++-
 5 files changed, 76 insertions(+), 72 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 5206041c..b2a63fdb 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -154,6 +154,10 @@ pub struct Args {
     #[arg(long, default_value_t = 0)]
     reclaim_rate_limit: usize,
 
+    /// (ms)
+    #[arg(long, default_value_t = 10)]
+    allocation_timeout: usize,
+
     /// `0` means equal to reclaimer count.
     #[arg(long, default_value_t = 0)]
     clean_region_threshold: usize,
@@ -339,6 +343,7 @@ async fn main() {
         reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
         event_listeners: vec![],
+        allocation_timeout: Duration::from_millis(args.allocation_timeout as u64),
         clean_region_threshold,
     };
 
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index e391cf8b..8a3fe099 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -149,7 +149,7 @@ where
             async move {
                 let mut iter = match RegionEntryIter::<K, V, D>::open(region).await {
                     Ok(Some(iter)) => iter,
-                    Ok(None) => return Ok(()),
+                    Ok(None) => return Ok(true),
                     Err(e) => return Err(e),
                 };
 
@@ -169,6 +169,7 @@ where
                         continue;
                     }
 
+                    // TODO(MrCroxx): Should reclaimer use wait if exceed limitation?
                     if let Some(rate) = rate.as_ref() && let Some(wait) = rate.consume(weight as f64) {
                         tokio::time::sleep(wait).await;
                     }
@@ -176,6 +177,10 @@ where
                     let mut writer = self.store.writer(key.clone(), weight);
                     writer.set_skippable();
 
+                    if !writer.judge() {
+                        continue;
+                    }
+
                     if writer.finish(value).await? {
                         for (index, reinsertion) in reinsertions.iter().enumerate() {
                             let judge = judges.get(index);
@@ -186,6 +191,10 @@ where
                             let judge = judges.get(index);
                             reinsertion.on_drop(&key, weight, &metrics, judge);
                         }
+                        // The writer is already been judged and admitted, but not inserted successfully and skipped.
+                        // That means allocating timeouts and there is no clean region available.
+                        // Reinsertion should be interrupted to make sure foreground insertion.
+                        return Ok(false);
                     }
 
                     metrics.op_bytes_reinsert.inc_by(weight as u64);
@@ -193,12 +202,20 @@ where
 
                 tracing::info!("[reclaimer] finish reinsertion, region: {}", region_id);
 
-                Ok(())
+                Ok(true)
             }
         };
 
-        if !self.store.reinsertions().is_empty() && let Err(e) = reinsert().await {
-            tracing::warn!("reinsert region {:?} error: {:?}", region, e);
+        if !self.store.reinsertions().is_empty() {
+            match reinsert().await {
+                Ok(true) => {
+                    tracing::info!("[reclaimer] reinsertion finish, region: {}", region_id)
+                }
+                Ok(false) => {
+                    tracing::info!("[reclaimer] reinsertion skipped, region: {}", region_id)
+                }
+                Err(e) => tracing::warn!("reinsert region {:?} error: {:?}", region, e),
+            }
         }
 
         // step 3: set region last block zero
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index d155b54c..738606ea 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -152,8 +152,7 @@ where
         let offset = inner.len;
         let region_id = self.id;
 
-        // reserve 1 align size for region footer
-        if inner.len + size + self.device.align() > inner.capacity {
+        if inner.len + size > inner.capacity {
             // if full, return the reserved 1 aligen write buf
             let remain = self.device.region_size() - inner.len;
             inner.len = self.device.region_size();
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index b9c31614..f18a4ecc 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -12,24 +12,22 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 
-use foyer_common::{
-    batch::{Batch, Identity},
-    queue::AsyncQueue,
-};
+use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{
     core::adapter::Link,
     eviction::{EvictionPolicy, EvictionPolicyExt},
     intrusive_adapter, key_adapter,
 };
-use parking_lot::{Mutex, RwLock};
+use parking_lot::RwLock;
+use tokio::sync::Mutex as AsyncMutex;
 use tracing::Instrument;
 
 use crate::{
     device::Device,
     metrics::Metrics,
-    region::{Region, RegionId, WriteSlice},
+    region::{AllocateResult, Region, RegionId, WriteSlice},
 };
 
 #[derive(Debug)]
@@ -56,8 +54,7 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    current: Mutex<Option<RegionId>>,
-    rotate_batch: Batch<(), ()>,
+    current: AsyncMutex<Option<RegionId>>,
 
     /// Buffer pool for dirty buffers.
     buffers: AsyncQueue<Vec<u8, D::IoBufferAllocator>>,
@@ -74,6 +71,8 @@ where
     /// Eviction policy.
     eviction: RwLock<EP>,
 
+    allocation_timeout: Duration,
+
     metrics: Arc<Metrics>,
 }
 
@@ -88,6 +87,7 @@ where
         region_count: usize,
         eviction_config: EP::Config,
         device: D,
+        allocation_timeout: Duration,
         metrics: Arc<Metrics>,
     ) -> Self {
         let buffers = AsyncQueue::new();
@@ -116,56 +116,45 @@ where
         }
 
         Self {
-            current: Mutex::new(None),
-            rotate_batch: Batch::new(),
+            current: AsyncMutex::new(None),
             buffers,
             clean_regions,
             dirty_regions,
             regions,
             items,
             eviction: RwLock::new(eviction),
+            allocation_timeout,
             metrics,
         }
     }
 
-    /// Allocate a buffer slice with given size in an active region to write.
     #[tracing::instrument(skip(self))]
     pub async fn allocate(&self, size: usize, must_allocate: bool) -> Option<WriteSlice> {
-        loop {
-            let res = self.allocate_inner(size);
-
-            if res.is_some() || !must_allocate {
-                return res;
+        let mut current = if must_allocate {
+            self.current.lock().await
+        } else {
+            match tokio::time::timeout(self.allocation_timeout, self.current.lock()).await {
+                Ok(current) => current,
+                Err(_) => return None,
             }
+        };
 
-            self.rotate().await;
-        }
-    }
-
-    #[tracing::instrument(skip(self))]
-    pub fn allocate_inner(&self, size: usize) -> Option<WriteSlice> {
-        let mut current = self.current.lock();
-
-        if let Some(region_id) = *current {
-            match self.region(&region_id).allocate(size) {
-                crate::region::AllocateResult::Ok(slice) => Some(slice),
-                crate::region::AllocateResult::Full { .. } => {
-                    self.dirty_regions.release(region_id);
-                    *current = None;
-                    None
+        loop {
+            if let Some(region_id) = *current {
+                match self.region(&region_id).allocate(size) {
+                    AllocateResult::Ok(slice) => return Some(slice),
+                    AllocateResult::Full { .. } => {
+                        self.dirty_regions.release(region_id);
+                        *current = None;
+                    }
+                    AllocateResult::None => {}
                 }
-                crate::region::AllocateResult::None => None,
             }
-        } else {
-            None
-        }
-    }
 
-    #[tracing::instrument(skip(self))]
-    pub async fn rotate(&self) {
-        match self.rotate_batch.push(()) {
-            Identity::Leader(rx) => {
-                // Wait a clean region to be released.
+            assert!(current.is_none());
+
+            // Wait a clean region to be released.
+            let region_id = {
                 let timer = self
                     .metrics
                     .inner_op_duration_acquire_clean_region
@@ -176,30 +165,21 @@ where
                     .instrument(tracing::debug_span!("acquire_clean_region"))
                     .await;
                 drop(timer);
+                region_id
+            };
 
-                tracing::info!("switch to clean region: {}", region_id);
-
-                let region = self.region(&region_id);
-                region.advance().await;
+            tracing::info!("switch to clean region: {}", region_id);
 
-                let buffer = self
-                    .buffers
-                    .acquire()
-                    .instrument(tracing::debug_span!("acquire_clean_buffer"))
-                    .await;
-                region.attach_buffer(buffer).await;
-
-                *self.current.lock() = Some(region_id);
+            let region = self.region(&region_id);
+            region.advance().await;
+            let buffer = self
+                .buffers
+                .acquire()
+                .instrument(tracing::debug_span!("acquire_clean_buffer"))
+                .await;
+            region.attach_buffer(buffer).await;
 
-                // Notify the batch to advance.
-                let items = self.rotate_batch.rotate();
-                tracing::Span::current().record("followers", items.len());
-                for item in items {
-                    item.tx.send(()).unwrap();
-                }
-                rx.await.unwrap();
-            }
-            Identity::Follower(rx) => rx.await.unwrap(),
+            *current = Some(region_id);
         }
     }
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index fa92beed..23dcab66 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -90,6 +90,9 @@ where
     /// Flush rate limits.
     pub reclaim_rate_limit: usize,
 
+    /// Allocation timout for skippable writers.
+    pub allocation_timeout: Duration,
+
     /// Clean region count threshold to trigger reclamation.
     ///
     /// `clean_region_threshold` is recommended to be equal or larger than `reclaimers`.
@@ -175,6 +178,7 @@ where
             device.regions(),
             config.eviction_config,
             device.clone(),
+            config.allocation_timeout,
             metrics.clone(),
         ));
 
@@ -311,7 +315,6 @@ where
         let mut writer = StoreWriter::new(self, key, weight);
         writer.set_force();
         let inserted = writer.finish(value).await?;
-        assert!(inserted);
         Ok(inserted)
     }
 
@@ -364,7 +367,6 @@ where
             }
         };
         let inserted = writer.finish(value).await?;
-        assert!(inserted);
         Ok(inserted)
     }
 
@@ -419,7 +421,6 @@ where
             }
         };
         let inserted = writer.finish(value).await?;
-        assert!(inserted);
         Ok(inserted)
     }
 
@@ -1229,6 +1230,7 @@ pub mod tests {
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
             event_listeners: vec![],
+            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
         };
 
@@ -1279,6 +1281,7 @@ pub mod tests {
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
             event_listeners: vec![],
+            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
         };
         let store = TestStore::open(config).await.unwrap();

From 1217ea94a8c207f8b9da7c519405c22141769150 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 25 Sep 2023 17:20:24 +0800
Subject: [PATCH 111/261] fix: fix recovery load unwrap (#142)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region.rs | 1 -
 foyer-storage/src/store.rs  | 6 ++++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 738606ea..1e45528a 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -153,7 +153,6 @@ where
         let region_id = self.id;
 
         if inner.len + size > inner.capacity {
-            // if full, return the reserved 1 aligen write buf
             let remain = self.device.region_size() - inner.len;
             inner.len = self.device.region_size();
             let range = inner.len - self.device.align()..inner.len;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 23dcab66..450a88f0 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -993,11 +993,13 @@ where
             return Ok(None);
         }
 
-        let slice = self
+        let Some(slice) = self
             .region
             .load(self.cursor..self.cursor + align, 0)
             .await?
-            .unwrap();
+        else {
+            return Ok(None);
+        };
 
         let Some(header) = EntryHeader::read(slice.as_ref()) else {
             return Ok(None);

From ac26695d5bf06fb4f2bd9aae2920382a1b37ebcd Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 25 Sep 2023 17:26:19 +0800
Subject: [PATCH 112/261] chore: bench support entry size range (#143)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index b2a63fdb..52f8fd28 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -22,6 +22,7 @@ mod utils;
 
 use std::{
     fs::create_dir_all,
+    ops::Range,
     path::PathBuf,
     sync::{
         atomic::{AtomicU64, Ordering},
@@ -49,7 +50,10 @@ use foyer_storage::{
 };
 use futures::future::join_all;
 use itertools::Itertools;
-use rand::{rngs::StdRng, Rng, SeedableRng};
+use rand::{
+    rngs::{OsRng, StdRng},
+    Rng, SeedableRng,
+};
 
 use export::MetricsExporter;
 use rate::RateLimiter;
@@ -92,7 +96,10 @@ pub struct Args {
     r_rate: f64,
 
     #[arg(long, default_value_t = 64 * 1024)]
-    entry_size: usize,
+    entry_size_min: usize,
+
+    #[arg(long, default_value_t = 64 * 1024)]
+    entry_size_max: usize,
 
     #[arg(long, default_value_t = 10000)]
     lookup_range: u64,
@@ -415,7 +422,7 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop_tx: broadc
     let w_handles = (0..args.writers)
         .map(|_| {
             tokio::spawn(write(
-                args.entry_size,
+                args.entry_size_min..args.entry_size_max + 1,
                 w_rate,
                 index.clone(),
                 store.clone(),
@@ -428,7 +435,6 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop_tx: broadc
     let r_handles = (0..args.readers)
         .map(|_| {
             tokio::spawn(read(
-                args.entry_size,
                 r_rate,
                 index.clone(),
                 store.clone(),
@@ -445,7 +451,7 @@ async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop_tx: broadc
 }
 
 async fn write(
-    entry_size: usize,
+    entry_size_range: Range<usize>,
     rate: Option<f64>,
     index: Arc<AtomicU64>,
     store: Arc<TStore>,
@@ -468,6 +474,7 @@ async fn write(
 
         let idx = index.fetch_add(1, Ordering::Relaxed);
         // TODO(MrCroxx): Use random content?
+        let entry_size = OsRng.gen_range(entry_size_range.clone());
         let data = vec![idx as u8; entry_size];
         if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
             tokio::time::sleep(wait).await;
@@ -489,9 +496,7 @@ async fn write(
     }
 }
 
-#[expect(clippy::too_many_arguments)]
 async fn read(
-    entry_size: usize,
     rate: Option<f64>,
     index: Arc<AtomicU64>,
     store: Arc<TStore>,
@@ -518,20 +523,21 @@ async fn read(
         let idx_max = index.load(Ordering::Relaxed);
         let idx = rng.gen_range(std::cmp::max(idx_max, look_up_range) - look_up_range..=idx_max);
 
-        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
-            tokio::time::sleep(wait).await;
-        }
-
         let time = Instant::now();
         let res = store.lookup(&idx).await.unwrap();
         let lat = time.elapsed().as_micros() as u64;
 
-        if res.is_some() {
-            assert_eq!(vec![idx as u8; entry_size], res.unwrap());
+        if let Some(buf) = res {
+            let entry_size = buf.len();
+            assert_eq!(vec![idx as u8; entry_size], buf);
             if let Err(e) = metrics.get_hit_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);
             }
             metrics.get_bytes.fetch_add(entry_size, Ordering::Relaxed);
+
+            if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
+                tokio::time::sleep(wait).await;
+            }
         } else {
             if let Err(e) = metrics.get_miss_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);

From d74c62980b6f51a671e26dec08f34e7bb0ff4c5f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 26 Sep 2023 14:40:31 +0800
Subject: [PATCH 113/261] refactor: refine erwlock (#144)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml         |  2 +-
 foyer-common/src/erwlock.rs     | 64 +++++++++++++++++++++++++++++
 foyer-common/src/lib.rs         |  1 +
 foyer-storage/src/region.rs     | 73 ++++++++++++---------------------
 foyer-workspace-hack/Cargo.toml |  1 -
 5 files changed, 92 insertions(+), 49 deletions(-)
 create mode 100644 foyer-common/src/erwlock.rs

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 1582fe6f..3e01d296 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -10,7 +10,7 @@ license = "Apache-2.0"
 [dependencies]
 bytes = "1"
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
-parking_lot = "0.12"
+parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 rand = "0.8.5"
 tokio = { version = "1", features = [
diff --git a/foyer-common/src/erwlock.rs b/foyer-common/src/erwlock.rs
new file mode 100644
index 00000000..b2acfb28
--- /dev/null
+++ b/foyer-common/src/erwlock.rs
@@ -0,0 +1,64 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use parking_lot::{
+    lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard,
+};
+use std::sync::Arc;
+
+pub trait ErwLockInner {
+    type R;
+    fn is_exclusive(&self, require: &Self::R) -> bool;
+}
+
+#[derive(Debug)]
+pub struct ErwLock<T: ErwLockInner> {
+    inner: Arc<RwLock<T>>,
+}
+
+impl<T: ErwLockInner> Clone for ErwLock<T> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<T: ErwLockInner> ErwLock<T> {
+    pub fn new(inner: T) -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(inner)),
+        }
+    }
+
+    pub fn read(&self) -> RwLockReadGuard<'_, T> {
+        self.inner.read()
+    }
+
+    pub fn write(&self) -> RwLockWriteGuard<'_, T> {
+        self.inner.write()
+    }
+
+    pub async fn exclusive(&self, require: &T::R) -> ArcRwLockWriteGuard<RawRwLock, T> {
+        loop {
+            {
+                let guard = self.inner.clone().write_arc();
+                if guard.is_exclusive(require) {
+                    return guard;
+                }
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(1)).await;
+        }
+    }
+}
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index d3208b94..4e3ec27a 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -19,6 +19,7 @@
 pub mod batch;
 pub mod bits;
 pub mod code;
+pub mod erwlock;
 pub mod queue;
 pub mod rate;
 pub mod rated_random;
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 1e45528a..1ba6d64b 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -12,12 +12,11 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, sync::Arc, task::Waker};
+use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, task::Waker};
 
 use bytes::{Buf, BufMut};
-use parking_lot::{
-    lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard,
-};
+use foyer_common::erwlock::{ErwLock, ErwLockInner};
+use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock};
 use tracing::instrument;
 
 use crate::{
@@ -84,6 +83,23 @@ where
     wakers: HashMap<usize, Waker>,
 }
 
+#[derive(Debug, Clone)]
+pub struct RegionInnerExclusiveRequire {
+    can_write: bool,
+    can_buffered_read: bool,
+    can_physical_read: bool,
+}
+
+impl<A: BufferAllocator> ErwLockInner for RegionInner<A> {
+    type R = RegionInnerExclusiveRequire;
+
+    fn is_exclusive(&self, require: &Self::R) -> bool {
+        (require.can_write || self.writers == 0)
+            && (require.can_buffered_read || self.buffered_readers == 0)
+            && (require.can_physical_read || self.physical_readers == 0)
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct Region<D>
 where
@@ -91,7 +107,7 @@ where
 {
     id: RegionId,
 
-    inner: ErwLock<D::IoBufferAllocator>,
+    inner: ErwLock<RegionInner<D::IoBufferAllocator>>,
 
     device: D,
 }
@@ -320,7 +336,11 @@ where
         can_physical_read: bool,
     ) -> ArcRwLockWriteGuard<RawRwLock, RegionInner<D::IoBufferAllocator>> {
         self.inner
-            .exclusive(can_write, can_buffered_read, can_physical_read)
+            .exclusive(&RegionInnerExclusiveRequire {
+                can_write,
+                can_buffered_read,
+                can_physical_read,
+            })
             .await
     }
 
@@ -525,44 +545,3 @@ where
         }
     }
 }
-
-#[derive(Debug, Clone)]
-pub struct ErwLock<A: BufferAllocator> {
-    inner: Arc<RwLock<RegionInner<A>>>,
-}
-
-impl<A: BufferAllocator> ErwLock<A> {
-    pub fn new(inner: RegionInner<A>) -> Self {
-        Self {
-            inner: Arc::new(RwLock::new(inner)),
-        }
-    }
-
-    pub fn read(&self) -> RwLockReadGuard<'_, RegionInner<A>> {
-        self.inner.read()
-    }
-
-    pub fn write(&self) -> RwLockWriteGuard<'_, RegionInner<A>> {
-        self.inner.write()
-    }
-
-    pub async fn exclusive(
-        &self,
-        can_write: bool,
-        can_buffered_read: bool,
-        can_physical_read: bool,
-    ) -> ArcRwLockWriteGuard<RawRwLock, RegionInner<A>> {
-        loop {
-            {
-                let guard = self.inner.clone().write_arc();
-                let is_ready = (can_write || guard.writers == 0)
-                    && (can_buffered_read || guard.buffered_readers == 0)
-                    && (can_physical_read || guard.physical_readers == 0);
-                if is_ready {
-                    return guard;
-                }
-            }
-            tokio::time::sleep(std::time::Duration::from_millis(1)).await;
-        }
-    }
-}
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 1e87c402..b35b807a 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -22,7 +22,6 @@ futures-util = { version = "0.3", default-features = false, features = ["async-a
 hyper = { version = "0.14", features = ["full"] }
 itertools = { version = "0.10" }
 libc = { version = "0.2", features = ["extra_traits"] }
-lock_api = { version = "0.4", features = ["arc_lock"] }
 memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }

From 6b590c16d547591137b8f4a658eba90c40eac9cb Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 26 Sep 2023 17:00:24 +0800
Subject: [PATCH 114/261] feat: support multiple allocators in round-robin
 order (#145)

* feat: support multiple allocators in round-robin order

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix deadlock

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml       |  4 +--
 .github/workflows/main.yml          |  4 +--
 .github/workflows/pull-request.yml  |  4 +--
 foyer-storage-bench/src/main.rs     | 27 +++++++++-------
 foyer-storage/src/region_manager.rs | 50 +++++++++++++++++++++++------
 foyer-storage/src/store.rs          | 34 +++++++++++++++-----
 6 files changed, 87 insertions(+), 36 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index cdf5a6f7..c55c64c4 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -111,7 +111,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -140,4 +140,4 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f75aef45..61fa082f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -118,7 +118,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -147,7 +147,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
 
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index b28bc1f6..e4f95dfa 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -117,7 +117,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -146,7 +146,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
 concurrency:
   group: environment-${{ github.ref }}
   cancel-in-progress: true
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 52f8fd28..a7c1fc38 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -74,10 +74,6 @@ pub struct Args {
     #[arg(short, long, default_value_t = 60)]
     time: u64,
 
-    /// must be power of 2
-    #[arg(long, default_value_t = 16)]
-    pools: usize,
-
     /// (s)
     #[arg(long, default_value_t = 2)]
     report_interval: u64,
@@ -168,12 +164,17 @@ pub struct Args {
     /// `0` means equal to reclaimer count.
     #[arg(long, default_value_t = 0)]
     clean_region_threshold: usize,
-}
 
-impl Args {
-    fn verify(&self) {
-        assert!(self.pools.is_power_of_two());
-    }
+    /// The count of allocators is `2 ^ allocator bits`.
+    ///
+    /// Note: The count of allocators should be greater than buffer count.
+    ///       (buffer count = buffer pool size / device region size)
+    #[arg(long, default_value_t = 0)]
+    allocator_bits: usize,
+
+    /// Weigher to enable metrics exporter.
+    #[arg(long, default_value_t = false)]
+    metrics: bool,
 }
 
 type TStore = LfuFsStore<u64, Vec<u8>>;
@@ -202,7 +203,7 @@ fn init_logger() {
         )]));
     let batch_config = BatchConfig::default()
         .with_max_queue_size(1048576)
-        .with_max_export_batch_size(1024)
+        .with_max_export_batch_size(4096)
         .with_max_concurrent_exports(4);
 
     let tracer = opentelemetry_otlp::new_pipeline()
@@ -267,9 +268,10 @@ async fn main() {
     }
 
     let args = Args::parse();
-    args.verify();
 
-    MetricsExporter::init("0.0.0.0:19970".parse().unwrap());
+    if args.metrics {
+        MetricsExporter::init("0.0.0.0:19970".parse().unwrap());
+    }
 
     println!("{:#?}", args);
 
@@ -341,6 +343,7 @@ async fn main() {
         name: "".to_string(),
         eviction_config,
         device_config,
+        allocator_bits: args.allocator_bits,
         admissions,
         reinsertions,
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index f18a4ecc..33fc6007 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -12,7 +12,13 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{sync::Arc, time::Duration};
+use std::{
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
 
 use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{
@@ -20,6 +26,7 @@ use foyer_intrusive::{
     eviction::{EvictionPolicy, EvictionPolicyExt},
     intrusive_adapter, key_adapter,
 };
+use itertools::Itertools;
 use parking_lot::RwLock;
 use tokio::sync::Mutex as AsyncMutex;
 use tracing::Instrument;
@@ -54,7 +61,9 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    current: AsyncMutex<Option<RegionId>>,
+    allocators: Vec<AsyncMutex<Option<Region<D>>>>,
+    allocator_bits: usize,
+    allocated: AtomicUsize,
 
     /// Buffer pool for dirty buffers.
     buffers: AsyncQueue<Vec<u8, D::IoBufferAllocator>>,
@@ -83,6 +92,7 @@ where
     EL: Link,
 {
     pub fn new(
+        allocator_bits: usize,
         buffer_count: usize,
         region_count: usize,
         eviction_config: EP::Config,
@@ -115,8 +125,14 @@ where
             items.push(item);
         }
 
+        let allocators = (0..(1 << allocator_bits))
+            .map(|_| AsyncMutex::new(None))
+            .collect_vec();
+
         Self {
-            current: AsyncMutex::new(None),
+            allocators,
+            allocated: AtomicUsize::new(0),
+            allocator_bits,
             buffers,
             clean_regions,
             dirty_regions,
@@ -130,21 +146,25 @@ where
 
     #[tracing::instrument(skip(self))]
     pub async fn allocate(&self, size: usize, must_allocate: bool) -> Option<WriteSlice> {
+        let allocated = self.allocated.fetch_add(1, Ordering::Relaxed);
+        let index = allocated & ((1 << self.allocator_bits) - 1);
+        let allocator = &self.allocators[index];
+
         let mut current = if must_allocate {
-            self.current.lock().await
+            allocator.lock().await
         } else {
-            match tokio::time::timeout(self.allocation_timeout, self.current.lock()).await {
+            match tokio::time::timeout(self.allocation_timeout, allocator.lock()).await {
                 Ok(current) => current,
                 Err(_) => return None,
             }
         };
 
         loop {
-            if let Some(region_id) = *current {
-                match self.region(&region_id).allocate(size) {
+            if let Some(region) = current.as_ref() {
+                match region.allocate(size) {
                     AllocateResult::Ok(slice) => return Some(slice),
                     AllocateResult::Full { .. } => {
-                        self.dirty_regions.release(region_id);
+                        self.dirty_regions.release(region.id());
                         *current = None;
                     }
                     AllocateResult::None => {}
@@ -168,7 +188,7 @@ where
                 region_id
             };
 
-            tracing::info!("switch to clean region: {}", region_id);
+            tracing::info!("allocator {} switch to clean region: {}", index, region_id);
 
             let region = self.region(&region_id);
             region.advance().await;
@@ -179,7 +199,17 @@ where
                 .await;
             region.attach_buffer(buffer).await;
 
-            *current = Some(region_id);
+            *current = Some(region.clone());
+        }
+    }
+
+    pub async fn seal(&self) {
+        for allocator in self.allocators.iter() {
+            let mut guard = allocator.lock().await;
+            if let Some(region) = guard.as_ref() {
+                self.dirty_regions.release(region.id());
+            }
+            *guard = None;
         }
     }
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 450a88f0..abcff361 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -69,6 +69,12 @@ where
     /// Device configurations.
     pub device_config: D::Config,
 
+    /// The count of allocators is `2 ^ allocator bits`.
+    ///
+    /// Note: The count of allocators should be greater than buffer count.
+    ///       (buffer count = buffer pool size / device region size)
+    pub allocator_bits: usize,
+
     /// Admission policies.
     pub admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
 
@@ -116,11 +122,18 @@ where
         f.debug_struct("StoreConfig")
             .field("eviction_config", &self.eviction_config)
             .field("device_config", &self.device_config)
+            .field("allocator_bits", &self.allocator_bits)
             .field("admissions", &self.admissions)
             .field("reinsertions", &self.reinsertions)
             .field("buffer_pool_size", &self.buffer_pool_size)
             .field("flushers", &self.flushers)
+            .field("flush_rate_limit", &self.flush_rate_limit)
             .field("reclaimers", &self.reclaimers)
+            .field("reclaim_rate_limit", &self.reclaim_rate_limit)
+            .field("allocation_timeout", &self.allocation_timeout)
+            .field("clean_region_threshold", &self.clean_region_threshold)
+            .field("recover_concurrency", &self.recover_concurrency)
+            .field("event_listeners", &self.event_listeners)
             .finish()
     }
 }
@@ -173,7 +186,15 @@ where
 
         let buffer_count = config.buffer_pool_size / device.region_size();
 
+        if buffer_count < (1 << config.allocator_bits) {
+            return Err(anyhow::anyhow!(
+                "The count of allocators shoule be greater than buffer count."
+            )
+            .into());
+        }
+
         let region_manager = Arc::new(RegionManager::new(
+            config.allocator_bits,
             buffer_count,
             device.regions(),
             config.eviction_config,
@@ -271,7 +292,7 @@ where
 
     pub async fn close(&self) -> Result<()> {
         // seal current dirty buffer and trigger flushing
-        self.seal().await?;
+        self.seal().await;
 
         // stop and wait for reclaimers
         let handles = self.reclaimer_handles.lock().drain(..).collect_vec();
@@ -555,13 +576,8 @@ where
         bits::align_up(self.device.align(), unaligned)
     }
 
-    async fn seal(&self) -> Result<()> {
-        // Try allocate the max size of a region to trigger flush.
-        // `max size == region size - region align` (first align block is reserved for region header)
-        self.region_manager
-            .allocate(self.device.region_size() - self.device.align(), false)
-            .await;
-        Ok(())
+    async fn seal(&self) {
+        self.region_manager.seal().await;
     }
 
     #[tracing::instrument(skip(self))]
@@ -1223,6 +1239,7 @@ pub mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
+            allocator_bits: 1,
             admissions,
             reinsertions,
             buffer_pool_size: 8 * MB,
@@ -1274,6 +1291,7 @@ pub mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
+            allocator_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,

From 80fe43490de24e412bc64c842239505a85c7ff52 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 6 Oct 2023 06:52:19 -0500
Subject: [PATCH 115/261] refactor: impl Debug for Store (#148)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/core/adapter.rs   |  8 +++++++-
 foyer-intrusive/src/eviction/fifo.rs  |  1 +
 foyer-intrusive/src/eviction/lfu.rs   |  1 +
 foyer-intrusive/src/eviction/lru.rs   |  1 +
 foyer-intrusive/src/eviction/mod.rs   |  2 +-
 foyer-intrusive/src/eviction/sfifo.rs |  1 +
 foyer-storage/src/lib.rs              | 16 +++++++++-------
 7 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index f226013e..db0fd0b7 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -41,7 +41,7 @@ pub trait Link: Send + Sync + 'static + Default + Debug {
 /// Pointer operations MUST be valid.
 ///
 /// [`Adapter`] is recommanded to be generated by macro `instrusive_adapter!`.
-pub unsafe trait Adapter: Send + Sync + 'static {
+pub unsafe trait Adapter: Send + Sync + Debug + 'static {
     type Pointer: Pointer;
     type Link: Link;
 
@@ -162,6 +162,12 @@ macro_rules! intrusive_adapter {
                 (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
             }
         }
+
+        impl<$($args),*> std::fmt::Debug for $name<$($args),*> $($where_)*{
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+                f.debug_struct(stringify!($name)).finish()
+            }
+        }
     };
     (
         $vis:vis $name:ident = $($rest:tt)*
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index bec8e750..c035fb9d 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -62,6 +62,7 @@ impl Link for FifoLink {
 intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DListLink } }
 
 /// FIFO policy
+#[derive(Debug)]
 pub struct Fifo<A>
 where
     A: Adapter<Link = FifoLink>,
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 6b37c490..6923ba00 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -124,6 +124,7 @@ intrusive_adapter! { LfuLinkMainDListAdapter = NonNull<LfuLink>: LfuLink { link_
 ///
 /// Tiny cache size:
 /// This default to 1%. There's no need to tune this parameter.
+#[derive(Debug)]
 pub struct Lfu<A>
 where
     A: Adapter<Link = LfuLink> + KeyAdapter<Link = LfuLink>,
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index dec035c1..2390db98 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -66,6 +66,7 @@ impl Link for LruLink {
 
 intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DListLink } }
 
+#[derive(Debug)]
 pub struct Lru<A>
 where
     A: Adapter<Link = LruLink>,
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index c9d28c7b..1e64e45d 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -17,7 +17,7 @@ use std::fmt::Debug;
 
 pub trait Config = Send + Sync + 'static + Debug + Clone;
 
-pub trait EvictionPolicy: Send + Sync + 'static {
+pub trait EvictionPolicy: Send + Sync + Debug + 'static {
     type Adapter: Adapter;
     type Config: Config;
 
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index 05722596..8d574799 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -103,6 +103,7 @@ intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: Segm
 /// of the system faster once it is beyond the pri-2 segment ratio. Segment
 /// ratio is put in place to prevent the lower segments getting so small a
 /// portion of the flash device.
+#[derive(Debug)]
 pub struct SegmentedFifo<A>
 where
     A: Adapter<Link = SegmentedFifoLink> + PriorityAdapter<Link = SegmentedFifoLink>,
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 92172d98..33f37c25 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -20,6 +20,8 @@
 #![feature(error_generic_member_access)]
 #![feature(lazy_cell)]
 #![feature(lint_reasons)]
+#![feature(async_fn_in_trait)]
+#![feature(return_position_impl_trait_in_trait)]
 
 pub mod admission;
 pub mod device;
@@ -74,21 +76,21 @@ pub type LfuFsStoreConfig<K, V> = store::StoreConfig<
     >,
 >;
 
-pub type SegmentedFifoFsStore<K, V> = store::Store<
+pub type FifoFsStore<K, V> = store::Store<
     K,
     V,
     device::fs::FsDevice,
-    foyer_intrusive::eviction::sfifo::SegmentedFifo<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::sfifo::SegmentedFifoLink>,
+    foyer_intrusive::eviction::fifo::Fifo<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
     >,
-    foyer_intrusive::eviction::sfifo::SegmentedFifoLink,
+    foyer_intrusive::eviction::fifo::FifoLink,
 >;
 
-pub type SegmentedFifoFsStoreConfig<K, V> = store::StoreConfig<
+pub type FifoFsStoreConfig<K, V> = store::StoreConfig<
     K,
     V,
     device::fs::FsDevice,
-    foyer_intrusive::eviction::sfifo::SegmentedFifo<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::sfifo::SegmentedFifoLink>,
+    foyer_intrusive::eviction::fifo::Fifo<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
     >,
 >;

From baab65df9555fb16a9c3ae01a07a0a15734b49b1 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 7 Oct 2023 02:44:04 -0500
Subject: [PATCH 116/261] refactor: extract Storage trait and remove event
 listener (#149)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |   3 +-
 foyer-storage/src/device/fs.rs  |   2 -
 foyer-storage/src/device/mod.rs |  22 +--
 foyer-storage/src/event.rs      |  47 ------
 foyer-storage/src/lib.rs        | 274 ++++++++++++++++++++++++++++++-
 foyer-storage/src/reclaimer.rs  |  12 +-
 foyer-storage/src/store.rs      | 276 ++++++++++----------------------
 7 files changed, 370 insertions(+), 266 deletions(-)
 delete mode 100644 foyer-storage/src/event.rs

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index a7c1fc38..583cd4ba 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -46,7 +46,7 @@ use foyer_storage::{
         ReinsertionPolicy,
     },
     store::StoreConfig,
-    LfuFsStore,
+    LfuFsStore, StorageExt,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -352,7 +352,6 @@ async fn main() {
         reclaimers: args.reclaimers,
         reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
-        event_listeners: vec![],
         allocation_timeout: Duration::from_millis(args.allocation_timeout as u64),
         clean_region_threshold,
     };
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 184a0643..32c03d56 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -27,7 +27,6 @@ use super::{
     error::{DeviceError, DeviceResult},
     Device, IoBuf, IoBufMut,
 };
-use async_trait::async_trait;
 use futures::future::try_join_all;
 use itertools::Itertools;
 
@@ -74,7 +73,6 @@ pub struct FsDevice {
     inner: Arc<FsDeviceInner>,
 }
 
-#[async_trait]
 impl Device for FsDevice {
     type Config = FsDeviceConfig;
     type IoBufferAllocator = AlignedAllocator;
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 85a10a16..91a50008 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -16,40 +16,43 @@ pub mod allocator;
 pub mod error;
 pub mod fs;
 
-use async_trait::async_trait;
 use std::{alloc::Allocator, fmt::Debug};
 
 use crate::region::RegionId;
 use error::DeviceResult;
+use futures::Future;
 
 pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
 pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
 pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
 
-#[async_trait]
 pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     type IoBufferAllocator: BufferAllocator;
-    type Config: Debug;
+    type Config: Send + Debug;
 
-    async fn open(config: Self::Config) -> DeviceResult<Self>;
+    #[must_use]
+    fn open(config: Self::Config) -> impl Future<Output = DeviceResult<Self>> + Send;
 
-    async fn write(
+    #[must_use]
+    fn write(
         &self,
         buf: impl IoBuf,
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> DeviceResult<usize>;
+    ) -> impl Future<Output = DeviceResult<usize>> + Send;
 
-    async fn read(
+    #[must_use]
+    fn read(
         &self,
         buf: impl IoBufMut,
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> DeviceResult<usize>;
+    ) -> impl Future<Output = DeviceResult<usize>> + Send;
 
-    async fn flush(&self) -> DeviceResult<()>;
+    #[must_use]
+    fn flush(&self) -> impl Future<Output = DeviceResult<()>> + Send;
 
     fn capacity(&self) -> usize;
 
@@ -94,7 +97,6 @@ pub mod tests {
         }
     }
 
-    #[async_trait]
     impl Device for NullDevice {
         type Config = usize;
         type IoBufferAllocator = AlignedAllocator;
diff --git a/foyer-storage/src/event.rs b/foyer-storage/src/event.rs
deleted file mode 100644
index 60cbd624..00000000
--- a/foyer-storage/src/event.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::fmt::Debug;
-
-use async_trait::async_trait;
-use foyer_common::code::{Key, Value};
-
-use crate::error::Result;
-
-#[expect(unused_variables)]
-#[async_trait]
-pub trait EventListener: Send + Sync + 'static + Debug {
-    type K: Key;
-    type V: Value;
-
-    async fn on_recover(&self, key: &Self::K) -> Result<()> {
-        Ok(())
-    }
-
-    async fn on_insert(&self, key: &Self::K) -> Result<()> {
-        Ok(())
-    }
-
-    async fn on_remove(&self, key: &Self::K) -> Result<()> {
-        Ok(())
-    }
-
-    async fn on_evict(&self, key: &Self::K) -> Result<()> {
-        Ok(())
-    }
-
-    async fn on_clear(&self) -> Result<()> {
-        Ok(())
-    }
-}
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 33f37c25..677149bb 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -22,11 +22,18 @@
 #![feature(lint_reasons)]
 #![feature(async_fn_in_trait)]
 #![feature(return_position_impl_trait_in_trait)]
+#![feature(associated_type_defaults)]
+
+use std::{fmt::Debug, sync::Arc};
+
+use foyer_common::code::{Key, Value};
+
+use error::Result;
+use futures::Future;
 
 pub mod admission;
 pub mod device;
 pub mod error;
-pub mod event;
 pub mod flusher;
 pub mod indices;
 pub mod judge;
@@ -94,3 +101,268 @@ pub type FifoFsStoreConfig<K, V> = store::StoreConfig<
         region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
     >,
 >;
+
+pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
+
+pub trait StorageWriter: Send + Sync + Debug {
+    type Key: Key;
+    type Value: Value;
+
+    fn judge(&mut self) -> bool;
+
+    fn finish(self, value: Self::Value) -> impl Future<Output = Result<bool>> + Send;
+}
+
+pub trait Storage: Send + Sync + Debug + 'static {
+    type Key: Key;
+    type Value: Value;
+    type Config: Send + Debug;
+    type Writer<'a>: StorageWriter<Key = Self::Key, Value = Self::Value>;
+
+    #[must_use]
+    fn open(config: Self::Config) -> impl Future<Output = Result<Arc<Self>>> + Send;
+
+    #[must_use]
+    fn close(&self) -> impl Future<Output = Result<()>> + Send;
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_>;
+
+    fn exists(&self, key: &Self::Key) -> Result<bool>;
+
+    #[must_use]
+    fn lookup(&self, key: &Self::Key) -> impl Future<Output = Result<Option<Self::Value>>> + Send;
+
+    fn remove(&self, key: &Self::Key) -> Result<bool>;
+
+    fn clear(&self) -> Result<()>;
+}
+
+pub trait StorageExt: Storage {
+    #[must_use]
+    #[tracing::instrument(skip(self, value))]
+    fn insert(
+        &self,
+        key: Self::Key,
+        value: Self::Value,
+    ) -> impl Future<Output = Result<bool>> + Send {
+        let weight = key.serialized_len() + value.serialized_len();
+        self.writer(key, weight).finish(value)
+    }
+
+    #[must_use]
+    #[tracing::instrument(skip(self, value))]
+    fn insert_if_not_exists(
+        &self,
+        key: Self::Key,
+        value: Self::Value,
+    ) -> impl Future<Output = Result<bool>> + Send {
+        async move {
+            if self.exists(&key)? {
+                return Ok(false);
+            }
+            self.insert(key, value).await
+        }
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[must_use]
+    #[tracing::instrument(skip(self, f))]
+    fn insert_with<F>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f() {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            writer.finish(value).await
+        }
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called to fetch value, and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    fn insert_with_future<F, FU>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> FU + Send,
+        FU: FetchValueFuture<Self::Value>,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f().await {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            writer.finish(value).await
+        }
+    }
+
+    #[tracing::instrument(skip(self, f))]
+    fn insert_if_not_exists_with<F>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
+    {
+        async move {
+            if self.exists(&key)? {
+                return Ok(false);
+            }
+            self.insert_with(key, f, weight).await
+        }
+    }
+
+    #[tracing::instrument(skip(self, f))]
+    fn insert_if_not_exists_with_future<F, FU>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> FU + Send,
+        FU: FetchValueFuture<Self::Value>,
+    {
+        async move {
+            if self.exists(&key)? {
+                return Ok(false);
+            }
+            self.insert_with_future(key, f, weight).await
+        }
+    }
+}
+
+impl<S: Storage> StorageExt for S {}
+
+pub trait ForceStorageWriter: StorageWriter {
+    fn set_force(&mut self);
+}
+
+pub trait ForceStorageExt: Storage
+where
+    for<'w> Self::Writer<'w>: ForceStorageWriter,
+{
+    #[tracing::instrument(skip(self, value))]
+    fn insert_force(
+        &self,
+        key: Self::Key,
+        value: Self::Value,
+    ) -> impl Future<Output = Result<bool>> + Send {
+        let weight = key.serialized_len() + value.serialized_len();
+        let mut writer = self.writer(key, weight);
+        writer.set_force();
+        writer.finish(value)
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    fn insert_force_with<F>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            writer.set_force();
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f() {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            let inserted = writer.finish(value).await?;
+            Ok(inserted)
+        }
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called to fetch value, and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    fn insert_force_with_future<F, FU>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> FU + Send,
+        FU: FetchValueFuture<Self::Value>,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            writer.set_force();
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f().await {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            let inserted = writer.finish(value).await?;
+            Ok(inserted)
+        }
+    }
+}
+
+impl<S> ForceStorageExt for S
+where
+    S: Storage,
+    for<'w> S::Writer<'w>: ForceStorageWriter,
+{
+}
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 8a3fe099..29f8df0a 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -17,7 +17,6 @@ use std::{sync::Arc, time::Duration};
 use crate::{
     device::Device,
     error::Result,
-    event::EventListener,
     judge::Judges,
     metrics::Metrics,
     region_manager::{RegionEpItemAdapter, RegionManager},
@@ -48,8 +47,6 @@ where
 
     rate_limiter: Option<Arc<RateLimiter>>,
 
-    event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
-
     metrics: Arc<Metrics>,
 
     stop_rx: broadcast::Receiver<()>,
@@ -68,7 +65,6 @@ where
         store: Arc<Store<K, V, D, EP, EL>>,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         rate_limiter: Option<Arc<RateLimiter>>,
-        event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
         metrics: Arc<Metrics>,
         stop_rx: broadcast::Receiver<()>,
     ) -> Self {
@@ -77,7 +73,6 @@ where
             store,
             region_manager,
             rate_limiter,
-            event_listeners,
             metrics,
             stop_rx,
         }
@@ -117,12 +112,7 @@ where
         let region = self.region_manager.region(&region_id);
 
         // step 1: drop indices
-        let indices = self.store.indices().take_region(&region_id);
-        for index in indices.iter() {
-            for listener in self.event_listeners.iter() {
-                listener.on_evict(&index.key).await?;
-            }
-        }
+        let _indices = self.store.indices().take_region(&region_id);
 
         // after drop indices and acquire exclusive lock, no writers or readers are supposed to access the region
         {
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index abcff361..40988048 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -33,7 +33,6 @@ use crate::{
     admission::AdmissionPolicy,
     device::Device,
     error::Result,
-    event::EventListener,
     flusher::Flusher,
     indices::{Index, Indices},
     judge::Judges,
@@ -42,6 +41,7 @@ use crate::{
     region::{Region, RegionHeader, RegionId, REGION_MAGIC},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
+    ForceStorageWriter, Storage, StorageWriter,
 };
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::core::adapter::Link;
@@ -106,9 +106,6 @@ where
 
     /// Concurrency of recovery.
     pub recover_concurrency: usize,
-
-    /// Event listsners.
-    pub event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
 }
 
 impl<K, V, D, EP> Debug for StoreConfig<K, V, D, EP>
@@ -133,7 +130,6 @@ where
             .field("allocation_timeout", &self.allocation_timeout)
             .field("clean_region_threshold", &self.clean_region_threshold)
             .field("recover_concurrency", &self.recover_concurrency)
-            .field("event_listeners", &self.event_listeners)
             .finish()
     }
 }
@@ -156,8 +152,6 @@ where
     admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
     reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
 
-    event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
-
     flusher_handles: Mutex<Vec<JoinHandle<()>>>,
     flushers_stop_tx: broadcast::Sender<()>,
 
@@ -221,7 +215,6 @@ where
             device: device.clone(),
             admissions: config.admissions,
             reinsertions: config.reinsertions,
-            event_listeners: config.event_listeners.clone(),
             flusher_handles: Mutex::new(vec![]),
             reclaimer_handles: Mutex::new(vec![]),
             flushers_stop_tx,
@@ -265,7 +258,6 @@ where
                     store.clone(),
                     region_manager.clone(),
                     reclaim_rate_limiter.clone(),
-                    config.event_listeners.clone(),
                     metrics.clone(),
                     stop_rx,
                 )
@@ -315,164 +307,6 @@ where
         Ok(())
     }
 
-    #[tracing::instrument(skip(self, value))]
-    pub async fn insert(&self, key: K, value: V) -> Result<bool> {
-        let weight = key.serialized_len() + value.serialized_len();
-        let writer = StoreWriter::new(self, key, weight);
-        writer.finish(value).await
-    }
-
-    #[tracing::instrument(skip(self, value))]
-    pub async fn insert_if_not_exists(&self, key: K, value: V) -> Result<bool> {
-        if self.exists(&key)? {
-            return Ok(false);
-        }
-        self.insert(key, value).await
-    }
-
-    #[tracing::instrument(skip(self, value))]
-    pub async fn insert_force(&self, key: K, value: V) -> Result<bool> {
-        let weight = key.serialized_len() + value.serialized_len();
-        let mut writer = StoreWriter::new(self, key, weight);
-        writer.set_force();
-        let inserted = writer.finish(value).await?;
-        Ok(inserted)
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self, f))]
-    pub async fn insert_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
-    where
-        F: FnOnce() -> anyhow::Result<V>,
-    {
-        let mut writer = self.writer(key, weight);
-        if !writer.judge() {
-            return Ok(false);
-        }
-        let value = match f() {
-            Ok(value) => value,
-            Err(e) => {
-                tracing::warn!("fetch value error: {:?}", e);
-                return Ok(false);
-            }
-        };
-        writer.finish(value).await
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self, f))]
-    pub async fn insert_force_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
-    where
-        F: FnOnce() -> anyhow::Result<V>,
-    {
-        let mut writer = self.writer(key, weight);
-        writer.set_force();
-        if !writer.judge() {
-            return Ok(false);
-        }
-        let value = match f() {
-            Ok(value) => value,
-            Err(e) => {
-                tracing::warn!("fetch value error: {:?}", e);
-                return Ok(false);
-            }
-        };
-        let inserted = writer.finish(value).await?;
-        Ok(inserted)
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called to fetch value, and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self, f))]
-    pub async fn insert_with_future<F, FU>(&self, key: K, f: F, weight: usize) -> Result<bool>
-    where
-        F: FnOnce() -> FU,
-        FU: FetchValueFuture<V>,
-    {
-        let mut writer = self.writer(key, weight);
-        if !writer.judge() {
-            return Ok(false);
-        }
-        let value = match f().await {
-            Ok(value) => value,
-            Err(e) => {
-                tracing::warn!("fetch value error: {:?}", e);
-                return Ok(false);
-            }
-        };
-        writer.finish(value).await
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called to fetch value, and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self, f))]
-    pub async fn insert_force_with_future<F, FU>(&self, key: K, f: F, weight: usize) -> Result<bool>
-    where
-        F: FnOnce() -> FU,
-        FU: FetchValueFuture<V>,
-    {
-        let mut writer = self.writer(key, weight);
-        writer.set_force();
-        if !writer.judge() {
-            return Ok(false);
-        }
-        let value = match f().await {
-            Ok(value) => value,
-            Err(e) => {
-                tracing::warn!("fetch value error: {:?}", e);
-                return Ok(false);
-            }
-        };
-        let inserted = writer.finish(value).await?;
-        Ok(inserted)
-    }
-
-    #[tracing::instrument(skip(self, f))]
-    pub async fn insert_if_not_exists_with<F>(&self, key: K, f: F, weight: usize) -> Result<bool>
-    where
-        F: FnOnce() -> anyhow::Result<V>,
-    {
-        if self.exists(&key)? {
-            return Ok(false);
-        }
-        self.insert_with(key, f, weight).await
-    }
-
-    #[tracing::instrument(skip(self, f))]
-    pub async fn insert_if_not_exists_with_future<F, FU>(
-        &self,
-        key: K,
-        f: F,
-        weight: usize,
-    ) -> Result<bool>
-    where
-        F: FnOnce() -> FU,
-        FU: FetchValueFuture<V>,
-    {
-        if self.exists(&key)? {
-            return Ok(false);
-        }
-        self.insert_with_future(key, f, weight).await
-    }
-
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[tracing::instrument(skip(self))]
     pub fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, D, EP, EL> {
@@ -535,28 +369,18 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    pub async fn remove(&self, key: &K) -> Result<bool> {
+    pub fn remove(&self, key: &K) -> Result<bool> {
         let _timer = self.metrics.op_duration_remove.start_timer();
 
         let res = self.indices.remove(key).is_some();
 
-        if res {
-            for listener in self.event_listeners.iter() {
-                listener.on_remove(key).await?;
-            }
-        }
-
         Ok(res)
     }
 
     #[tracing::instrument(skip(self))]
-    pub async fn clear(&self) -> Result<()> {
+    pub fn clear(&self) -> Result<()> {
         self.indices.clear();
 
-        for listener in self.event_listeners.iter() {
-            listener.on_clear().await?;
-        }
-
         // TODO(MrCroxx): set all regions as clean?
 
         Ok(())
@@ -592,11 +416,9 @@ where
             let irx = rx.clone();
             let region_manager = self.region_manager.clone();
             let indices = self.indices.clone();
-            let event_listeners = self.event_listeners.clone();
             let handle = tokio::spawn(async move {
                 itx.send(()).await.unwrap();
-                let res =
-                    Self::recover_region(region_id, region_manager, indices, event_listeners).await;
+                let res = Self::recover_region(region_id, region_manager, indices).await;
                 irx.recv().await.unwrap();
                 res
             });
@@ -632,14 +454,10 @@ where
         region_id: RegionId,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         indices: Arc<Indices<K>>,
-        event_listeners: Vec<Arc<dyn EventListener<K = K, V = V>>>,
     ) -> Result<bool> {
         let region = region_manager.region(&region_id).clone();
         let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
             while let Some(index) = iter.next().await? {
-                for listener in event_listeners.iter() {
-                    listener.on_recover(&index.key).await?;
-                }
                 indices.insert(index);
             }
             region_manager.eviction_push(region_id);
@@ -717,10 +535,6 @@ where
 
         self.indices.insert(index);
 
-        for listener in self.event_listeners.iter() {
-            listener.on_insert(key).await?;
-        }
-
         let duration = now.elapsed() + writer.duration;
         self.metrics
             .op_duration_insert_inserted
@@ -1094,13 +908,91 @@ where
     }
 }
 
+impl<'a, K, V, D, EP, EL> StorageWriter for StoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    type Key = K;
+    type Value = V;
+
+    fn judge(&mut self) -> bool {
+        self.judge()
+    }
+
+    async fn finish(self, value: Self::Value) -> Result<bool> {
+        self.finish(value).await
+    }
+}
+
+impl<'a, K, V, D, EP, EL> ForceStorageWriter for StoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    fn set_force(&mut self) {
+        self.set_force()
+    }
+}
+
+impl<K, V, D, EP, EL> Storage for Store<K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    type Key = K;
+    type Value = V;
+    type Config = StoreConfig<K, V, D, EP>;
+    type Writer<'a> = StoreWriter<'a, K, V, D, EP, EL>;
+
+    async fn open(config: Self::Config) -> Result<Arc<Self>> {
+        Self::open(config).await
+    }
+
+    async fn close(&self) -> Result<()> {
+        todo!()
+    }
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+        self.writer(key, weight)
+    }
+
+    fn exists(&self, key: &Self::Key) -> Result<bool> {
+        self.exists(key)
+    }
+
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        self.lookup(key).await
+    }
+
+    fn remove(&self, key: &Self::Key) -> Result<bool> {
+        self.remove(key)
+    }
+
+    fn clear(&self) -> Result<()> {
+        self.clear()
+    }
+}
+
 #[cfg(test)]
 pub mod tests {
     use std::{collections::HashSet, path::PathBuf};
 
     use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
 
-    use crate::device::fs::{FsDevice, FsDeviceConfig};
+    use crate::{
+        device::fs::{FsDevice, FsDeviceConfig},
+        StorageExt,
+    };
 
     use super::*;
 
@@ -1248,7 +1140,6 @@ pub mod tests {
             reclaimers: 1,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            event_listeners: vec![],
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
         };
@@ -1300,7 +1191,6 @@ pub mod tests {
             reclaimers: 0,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            event_listeners: vec![],
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
         };

From 5a8aeefbeae1d613dc876ab2abfb3ec9aea18e67 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 7 Oct 2023 02:50:53 -0500
Subject: [PATCH 117/261] chore: add udeps check (#150)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                       | 1 +
 foyer-common/Cargo.toml        | 3 +++
 foyer-intrusive/Cargo.toml     | 4 +++-
 foyer-memory/Cargo.toml        | 4 +++-
 foyer-storage-bench/Cargo.toml | 3 +++
 foyer-storage/Cargo.toml       | 4 +++-
 foyer/Cargo.toml               | 3 +++
 7 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index f9238be3..ad07a9a4 100644
--- a/Makefile
+++ b/Makefile
@@ -14,6 +14,7 @@ check:
 	cargo clippy --all-targets --features tokio-console
 	cargo clippy --all-targets --features trace
 	cargo clippy --all-targets
+	cargo udeps --workspace --exclude foyer-workspace-hack
 
 test:
 	RUST_BACKTRACE=1 cargo nextest run --all
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 3e01d296..a58a1e36 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -7,6 +7,9 @@ description = "Hybrid cache for Rust"
 license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
+
 [dependencies]
 bytes = "1"
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 2f9ea412..75eaad91 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -7,8 +7,10 @@ description = "Hybrid cache for Rust"
 license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-[dependencies]
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
 
+[dependencies]
 bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 0a39bfb0..298ef90d 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -7,13 +7,15 @@ description = "Hybrid cache for Rust"
 license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
+
 [dependencies]
 bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
-itertools = "0.11.0"
 libc = "0.2"
 memoffset = "0.9"
 parking_lot = "0.12"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index f04d5160..195e6fcd 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -7,6 +7,9 @@ description = "Hybrid cache for Rust"
 license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
+
 [dependencies]
 anyhow = "1"
 bytesize = "1"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index f88ce906..f31dd1f9 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -7,10 +7,12 @@ description = "Hybrid cache for Rust"
 license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
+
 [dependencies]
 anyhow = "1.0"
 async-channel = "1.8"
-async-trait = "0.1"
 bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 901bafcc..675477f7 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -7,6 +7,9 @@ description = "Hybrid cache for Rust"
 license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
+
 [dependencies]
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }

From 7e57346d628213a9070302e50686e611aead747b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 7 Oct 2023 05:34:43 -0500
Subject: [PATCH 118/261] feat: introduce lazy store (#151)

* feat: introduce lazy store

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* use from instead of into

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* make impl fn private

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix once lock clone and add test for lazy store

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |   2 +-
 foyer-storage/src/lib.rs        | 485 +++++++++++++++++++++++++++++++-
 foyer-storage/src/reclaimer.rs  |   1 +
 foyer-storage/src/store.rs      |  19 +-
 4 files changed, 495 insertions(+), 12 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 583cd4ba..e730e5ea 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -46,7 +46,7 @@ use foyer_storage::{
         ReinsertionPolicy,
     },
     store::StoreConfig,
-    LfuFsStore, StorageExt,
+    LfuFsStore, Storage, StorageExt,
 };
 use futures::future::join_all;
 use itertools::Itertools;
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 677149bb..40460247 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -24,7 +24,10 @@
 #![feature(return_position_impl_trait_in_trait)]
 #![feature(associated_type_defaults)]
 
-use std::{fmt::Debug, sync::Arc};
+use std::{
+    fmt::Debug,
+    sync::{Arc, OnceLock},
+};
 
 use foyer_common::code::{Key, Value};
 
@@ -64,6 +67,17 @@ pub type LruFsStoreConfig<K, V> = store::StoreConfig<
     >,
 >;
 
+pub type LruFsStoreWriter<'w, K, V> = store::StoreWriter<
+    'w,
+    K,
+    V,
+    device::fs::FsDevice,
+    foyer_intrusive::eviction::lru::Lru<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
+    >,
+    foyer_intrusive::eviction::lru::LruLink,
+>;
+
 pub type LfuFsStore<K, V> = store::Store<
     K,
     V,
@@ -83,6 +97,17 @@ pub type LfuFsStoreConfig<K, V> = store::StoreConfig<
     >,
 >;
 
+pub type LfuFsStoreWriter<'w, K, V> = store::StoreWriter<
+    'w,
+    K,
+    V,
+    device::fs::FsDevice,
+    foyer_intrusive::eviction::lfu::Lfu<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
+    >,
+    foyer_intrusive::eviction::lfu::LfuLink,
+>;
+
 pub type FifoFsStore<K, V> = store::Store<
     K,
     V,
@@ -102,6 +127,17 @@ pub type FifoFsStoreConfig<K, V> = store::StoreConfig<
     >,
 >;
 
+pub type FifoFsStoreWriter<'w, K, V> = store::StoreWriter<
+    'w,
+    K,
+    V,
+    device::fs::FsDevice,
+    foyer_intrusive::eviction::fifo::Fifo<
+        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
+    >,
+    foyer_intrusive::eviction::fifo::FifoLink,
+>;
+
 pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
 
 pub trait StorageWriter: Send + Sync + Debug {
@@ -117,10 +153,11 @@ pub trait Storage: Send + Sync + Debug + 'static {
     type Key: Key;
     type Value: Value;
     type Config: Send + Debug;
+    type Owned: Send + Sync + Debug + 'static;
     type Writer<'a>: StorageWriter<Key = Self::Key, Value = Self::Value>;
 
     #[must_use]
-    fn open(config: Self::Config) -> impl Future<Output = Result<Arc<Self>>> + Send;
+    fn open(config: Self::Config) -> impl Future<Output = Result<Self::Owned>> + Send;
 
     #[must_use]
     fn close(&self) -> impl Future<Output = Result<()>> + Send;
@@ -366,3 +403,447 @@ where
     for<'w> S::Writer<'w>: ForceStorageWriter,
 {
 }
+
+#[derive(Debug)]
+pub enum StoreConfig<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    LruFsStoreConfig { config: LruFsStoreConfig<K, V> },
+    LfuFsStoreConfig { config: LfuFsStoreConfig<K, V> },
+    FifoFsStoreConfig { config: FifoFsStoreConfig<K, V> },
+    None,
+}
+
+impl<K, V> From<LruFsStoreConfig<K, V>> for StoreConfig<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(config: LruFsStoreConfig<K, V>) -> Self {
+        StoreConfig::LruFsStoreConfig { config }
+    }
+}
+
+impl<K, V> From<LfuFsStoreConfig<K, V>> for StoreConfig<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(config: LfuFsStoreConfig<K, V>) -> Self {
+        StoreConfig::LfuFsStoreConfig { config }
+    }
+}
+
+impl<K, V> From<FifoFsStoreConfig<K, V>> for StoreConfig<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(config: FifoFsStoreConfig<K, V>) -> Self {
+        StoreConfig::FifoFsStoreConfig { config }
+    }
+}
+
+#[derive(Debug)]
+pub enum StoreWriter<'a, K, V>
+where
+    K: Key,
+    V: Value,
+{
+    LruFsStorWriter { writer: LruFsStoreWriter<'a, K, V> },
+    LfuFsStorWriter { writer: LfuFsStoreWriter<'a, K, V> },
+    FifoFsStoreWriter { writer: FifoFsStoreWriter<'a, K, V> },
+    None,
+}
+
+impl<'a, K, V> From<LruFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(writer: LruFsStoreWriter<'a, K, V>) -> Self {
+        StoreWriter::LruFsStorWriter { writer }
+    }
+}
+
+impl<'a, K, V> From<LfuFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(writer: LfuFsStoreWriter<'a, K, V>) -> Self {
+        StoreWriter::LfuFsStorWriter { writer }
+    }
+}
+
+impl<'a, K, V> From<FifoFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(writer: FifoFsStoreWriter<'a, K, V>) -> Self {
+        StoreWriter::FifoFsStoreWriter { writer }
+    }
+}
+
+#[derive(Debug)]
+pub enum Store<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    LruFsStore { store: Arc<LruFsStore<K, V>> },
+    LfuFsStore { store: Arc<LfuFsStore<K, V>> },
+    FifoFsStore { store: Arc<FifoFsStore<K, V>> },
+    None,
+}
+
+impl<K, V> Clone for Store<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn clone(&self) -> Self {
+        match self {
+            Self::LruFsStore { store } => Self::LruFsStore {
+                store: Arc::clone(store),
+            },
+            Self::LfuFsStore { store } => Self::LfuFsStore {
+                store: Arc::clone(store),
+            },
+            Self::FifoFsStore { store } => Self::FifoFsStore {
+                store: Arc::clone(store),
+            },
+            Self::None => Self::None,
+        }
+    }
+}
+
+impl<'a, K, V> StorageWriter for StoreWriter<'a, K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+
+    fn judge(&mut self) -> bool {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.judge(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.judge(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.judge(),
+            StoreWriter::None => false,
+        }
+    }
+
+    async fn finish(self, value: Self::Value) -> Result<bool> {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.finish(value).await,
+            StoreWriter::LfuFsStorWriter { writer } => writer.finish(value).await,
+            StoreWriter::FifoFsStoreWriter { writer } => writer.finish(value).await,
+            StoreWriter::None => Ok(false),
+        }
+    }
+}
+
+impl<'a, K, V> ForceStorageWriter for StoreWriter<'a, K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn set_force(&mut self) {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.set_force(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.set_force(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.set_force(),
+            StoreWriter::None => {}
+        }
+    }
+}
+
+impl<K, V> Storage for Store<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Config = StoreConfig<K, V>;
+    type Owned = Self;
+    type Writer<'a> = StoreWriter<'a, K, V>;
+
+    async fn open(config: Self::Config) -> Result<Self::Owned> {
+        match config {
+            StoreConfig::LruFsStoreConfig { config } => {
+                let store = LruFsStore::open(config).await?;
+                Ok(Self::LruFsStore { store })
+            }
+            StoreConfig::LfuFsStoreConfig { config } => {
+                let store = LfuFsStore::open(config).await?;
+                Ok(Self::LfuFsStore { store })
+            }
+            StoreConfig::FifoFsStoreConfig { config } => {
+                let store = FifoFsStore::open(config).await?;
+                Ok(Self::FifoFsStore { store })
+            }
+            StoreConfig::None => Ok(Self::None),
+        }
+    }
+
+    async fn close(&self) -> Result<()> {
+        match self {
+            Store::LruFsStore { store } => store.close().await,
+            Store::LfuFsStore { store } => store.close().await,
+            Store::FifoFsStore { store } => store.close().await,
+            Store::None => Ok(()),
+        }
+    }
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+        match self {
+            Store::LruFsStore { store } => store.writer(key, weight).into(),
+            Store::LfuFsStore { store } => store.writer(key, weight).into(),
+            Store::FifoFsStore { store } => store.writer(key, weight).into(),
+            Store::None => StoreWriter::None,
+        }
+    }
+
+    fn exists(&self, key: &Self::Key) -> Result<bool> {
+        match self {
+            Store::LruFsStore { store } => store.exists(key),
+            Store::LfuFsStore { store } => store.exists(key),
+            Store::FifoFsStore { store } => store.exists(key),
+            Store::None => Ok(false),
+        }
+    }
+
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        match self {
+            Store::LruFsStore { store } => store.lookup(key).await,
+            Store::LfuFsStore { store } => store.lookup(key).await,
+            Store::FifoFsStore { store } => store.lookup(key).await,
+            Store::None => Ok(None),
+        }
+    }
+
+    fn remove(&self, key: &Self::Key) -> Result<bool> {
+        match self {
+            Store::LruFsStore { store } => store.remove(key),
+            Store::LfuFsStore { store } => store.remove(key),
+            Store::FifoFsStore { store } => store.remove(key),
+            Store::None => Ok(false),
+        }
+    }
+
+    fn clear(&self) -> Result<()> {
+        match self {
+            Store::LruFsStore { store } => store.clear(),
+            Store::LfuFsStore { store } => store.clear(),
+            Store::FifoFsStore { store } => store.clear(),
+            Store::None => Ok(()),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct LazyStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    once: Arc<OnceLock<Store<K, V>>>,
+    none: Store<K, V>,
+}
+
+impl<K, V> LazyStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn lazy(config: StoreConfig<K, V>) -> Self {
+        let (res, task) = Self::lazy_with_task(config);
+
+        tokio::spawn(task);
+
+        res
+    }
+
+    pub fn lazy_with_task(
+        config: StoreConfig<K, V>,
+    ) -> (Self, impl Future<Output = Result<Store<K, V>>> + Send) {
+        let once = Arc::new(OnceLock::new());
+
+        let task = {
+            let once = once.clone();
+            async move {
+                let store = match Store::open(config).await {
+                    Ok(store) => store,
+                    Err(e) => {
+                        tracing::error!("Lazy open store fail: {}", e);
+                        return Err(e);
+                    }
+                };
+                once.set(store.clone()).unwrap();
+                Ok(store)
+            }
+        };
+
+        let res = Self {
+            once,
+            none: Store::None,
+        };
+
+        (res, task)
+    }
+}
+
+impl<K, V> Storage for LazyStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Config = StoreConfig<K, V>;
+    type Owned = Self;
+    type Writer<'a> = StoreWriter<'a, K, V>;
+
+    async fn open(config: Self::Config) -> Result<Self::Owned> {
+        let once = Arc::new(OnceLock::new());
+        let store = Store::open(config).await?;
+        once.set(store).unwrap();
+        Ok(Self {
+            once,
+            none: Store::None,
+        })
+    }
+
+    async fn close(&self) -> Result<()> {
+        match self.once.get() {
+            Some(store) => store.close().await,
+            None => self.none.close().await,
+        }
+    }
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+        match self.once.get() {
+            Some(store) => store.writer(key, weight),
+            None => self.none.writer(key, weight),
+        }
+    }
+
+    fn exists(&self, key: &Self::Key) -> Result<bool> {
+        match self.once.get() {
+            Some(store) => store.exists(key),
+            None => self.none.exists(key),
+        }
+    }
+
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        match self.once.get() {
+            Some(store) => store.lookup(key).await,
+            None => self.none.lookup(key).await,
+        }
+    }
+
+    fn remove(&self, key: &Self::Key) -> Result<bool> {
+        match self.once.get() {
+            Some(store) => store.remove(key),
+            None => self.none.remove(key),
+        }
+    }
+
+    fn clear(&self) -> Result<()> {
+        match self.once.get() {
+            Some(store) => store.clear(),
+            None => self.none.clear(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{path::PathBuf, time::Duration};
+
+    use foyer_intrusive::eviction::fifo::FifoConfig;
+
+    use crate::device::fs::FsDeviceConfig;
+
+    use super::*;
+
+    const KB: usize = 1024;
+    const MB: usize = 1024 * 1024;
+
+    #[tokio::test]
+    async fn test_lazy_store() {
+        let tempdir = tempfile::tempdir().unwrap();
+
+        let config = FifoFsStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            allocator_bits: 1,
+            admissions: vec![],
+            reinsertions: vec![],
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 1,
+            reclaim_rate_limit: 0,
+            recover_concurrency: 2,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+        };
+
+        let (store, task) = LazyStore::lazy_with_task(config.into());
+
+        assert!(!store.insert(100, 100).await.unwrap());
+
+        tokio::spawn(task).await.unwrap().unwrap();
+
+        assert!(store.insert(100, 100).await.unwrap());
+        assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
+
+        store.close().await.unwrap();
+        drop(store);
+
+        let config = FifoFsStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            allocator_bits: 1,
+            admissions: vec![],
+            reinsertions: vec![],
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 1,
+            reclaim_rate_limit: 0,
+            recover_concurrency: 2,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+        };
+
+        let (store, task) = LazyStore::lazy_with_task(config.into());
+
+        assert!(store.lookup(&100).await.unwrap().is_none());
+
+        tokio::spawn(task).await.unwrap().unwrap();
+
+        assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
+    }
+}
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 29f8df0a..24ced44d 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -21,6 +21,7 @@ use crate::{
     metrics::Metrics,
     region_manager::{RegionEpItemAdapter, RegionManager},
     store::{RegionEntryIter, Store},
+    Storage,
 };
 use bytes::BufMut;
 use foyer_common::{
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 40988048..14719615 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -171,7 +171,7 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    pub async fn open(config: StoreConfig<K, V, D, EP>) -> Result<Arc<Self>> {
+    async fn open(config: StoreConfig<K, V, D, EP>) -> Result<Arc<Self>> {
         tracing::info!("open store with config:\n{:#?}", config);
 
         let metrics = Arc::new(METRICS.foyer(&config.name));
@@ -282,7 +282,7 @@ where
         Ok(store)
     }
 
-    pub async fn close(&self) -> Result<()> {
+    async fn close(&self) -> Result<()> {
         // seal current dirty buffer and trigger flushing
         self.seal().await;
 
@@ -309,17 +309,17 @@ where
 
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[tracing::instrument(skip(self))]
-    pub fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, D, EP, EL> {
+    fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, D, EP, EL> {
         StoreWriter::new(self, key, weight)
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn exists(&self, key: &K) -> Result<bool> {
+    fn exists(&self, key: &K) -> Result<bool> {
         Ok(self.indices.lookup(key).is_some())
     }
 
     #[tracing::instrument(skip(self))]
-    pub async fn lookup(&self, key: &K) -> Result<Option<V>> {
+    async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
 
         let index = match self.indices.lookup(key) {
@@ -369,7 +369,7 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn remove(&self, key: &K) -> Result<bool> {
+    fn remove(&self, key: &K) -> Result<bool> {
         let _timer = self.metrics.op_duration_remove.start_timer();
 
         let res = self.indices.remove(key).is_some();
@@ -378,7 +378,7 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn clear(&self) -> Result<()> {
+    fn clear(&self) -> Result<()> {
         self.indices.clear();
 
         // TODO(MrCroxx): set all regions as clean?
@@ -952,14 +952,15 @@ where
     type Key = K;
     type Value = V;
     type Config = StoreConfig<K, V, D, EP>;
+    type Owned = Arc<Self>;
     type Writer<'a> = StoreWriter<'a, K, V, D, EP, EL>;
 
-    async fn open(config: Self::Config) -> Result<Arc<Self>> {
+    async fn open(config: Self::Config) -> Result<Self::Owned> {
         Self::open(config).await
     }
 
     async fn close(&self) -> Result<()> {
-        todo!()
+        self.close().await
     }
 
     fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {

From fc511d32cda0dd8473dddd5fc9462b831a3aa505 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 7 Oct 2023 09:41:04 -0500
Subject: [PATCH 119/261] refactor: remod (#152)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |    7 +-
 foyer-storage/src/generic.rs    | 1217 ++++++++++++++++++++++++++++++
 foyer-storage/src/lazy.rs       |  222 ++++++
 foyer-storage/src/lib.rs        |  813 +-------------------
 foyer-storage/src/reclaimer.rs  |    8 +-
 foyer-storage/src/storage.rs    |  286 ++++++++
 foyer-storage/src/store.rs      | 1221 ++++---------------------------
 7 files changed, 1889 insertions(+), 1885 deletions(-)
 create mode 100644 foyer-storage/src/generic.rs
 create mode 100644 foyer-storage/src/lazy.rs
 create mode 100644 foyer-storage/src/storage.rs

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index e730e5ea..f20784da 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -41,12 +41,13 @@ use foyer_storage::{
         AdmissionPolicy,
     },
     device::fs::FsDeviceConfig,
+    generic::GenericStoreConfig,
     reinsertion::{
         rated_random::RatedRandomReinsertionPolicy, rated_ticket::RatedTicketReinsertionPolicy,
         ReinsertionPolicy,
     },
-    store::StoreConfig,
-    LfuFsStore, Storage, StorageExt,
+    storage::{Storage, StorageExt},
+    store::LfuFsStore,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -339,7 +340,7 @@ async fn main() {
         args.clean_region_threshold
     };
 
-    let config = StoreConfig {
+    let config = GenericStoreConfig {
         name: "".to_string(),
         eviction_config,
         device_config,
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
new file mode 100644
index 00000000..a9190a16
--- /dev/null
+++ b/foyer-storage/src/generic.rs
@@ -0,0 +1,1217 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use bitmaps::Bitmap;
+use bytes::{Buf, BufMut};
+use foyer_common::{bits, rate::RateLimiter};
+use foyer_intrusive::eviction::EvictionPolicy;
+use futures::{future::try_join_all, Future};
+use itertools::Itertools;
+use parking_lot::Mutex;
+use tokio::{sync::broadcast, task::JoinHandle};
+use twox_hash::XxHash64;
+
+use crate::{
+    admission::AdmissionPolicy,
+    device::Device,
+    error::Result,
+    flusher::Flusher,
+    indices::{Index, Indices},
+    judge::Judges,
+    metrics::{Metrics, METRICS},
+    reclaimer::Reclaimer,
+    region::{Region, RegionHeader, RegionId, REGION_MAGIC},
+    region_manager::{RegionEpItemAdapter, RegionManager},
+    reinsertion::ReinsertionPolicy,
+    storage::{ForceStorageWriter, Storage, StorageWriter},
+};
+use foyer_common::code::{Key, Value};
+use foyer_intrusive::core::adapter::Link;
+use std::hash::Hasher;
+
+const DEFAULT_BROADCAST_CAPACITY: usize = 4096;
+
+pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
+
+pub struct GenericStoreConfig<K, V, D, EP>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy,
+{
+    /// For distinguish different foyer metrics.
+    ///
+    /// Metrics of this foyer instance has label `foyer = {{ name }}`.
+    pub name: String,
+
+    /// Evictino policy configurations.
+    pub eviction_config: EP::Config,
+
+    /// Device configurations.
+    pub device_config: D::Config,
+
+    /// The count of allocators is `2 ^ allocator bits`.
+    ///
+    /// Note: The count of allocators should be greater than buffer count.
+    ///       (buffer count = buffer pool size / device region size)
+    pub allocator_bits: usize,
+
+    /// Admission policies.
+    pub admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
+
+    /// Reinsertion policies.
+    pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
+
+    /// Buffer pool size, should be a multiplier of device region size.
+    pub buffer_pool_size: usize,
+
+    /// Count of flushers.
+    pub flushers: usize,
+
+    /// Flush rate limits.
+    pub flush_rate_limit: usize,
+
+    /// Count of reclaimers.
+    pub reclaimers: usize,
+
+    /// Flush rate limits.
+    pub reclaim_rate_limit: usize,
+
+    /// Allocation timout for skippable writers.
+    pub allocation_timeout: Duration,
+
+    /// Clean region count threshold to trigger reclamation.
+    ///
+    /// `clean_region_threshold` is recommended to be equal or larger than `reclaimers`.
+    pub clean_region_threshold: usize,
+
+    /// Concurrency of recovery.
+    pub recover_concurrency: usize,
+}
+
+impl<K, V, D, EP> Debug for GenericStoreConfig<K, V, D, EP>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("StoreConfig")
+            .field("eviction_config", &self.eviction_config)
+            .field("device_config", &self.device_config)
+            .field("allocator_bits", &self.allocator_bits)
+            .field("admissions", &self.admissions)
+            .field("reinsertions", &self.reinsertions)
+            .field("buffer_pool_size", &self.buffer_pool_size)
+            .field("flushers", &self.flushers)
+            .field("flush_rate_limit", &self.flush_rate_limit)
+            .field("reclaimers", &self.reclaimers)
+            .field("reclaim_rate_limit", &self.reclaim_rate_limit)
+            .field("allocation_timeout", &self.allocation_timeout)
+            .field("clean_region_threshold", &self.clean_region_threshold)
+            .field("recover_concurrency", &self.recover_concurrency)
+            .finish()
+    }
+}
+
+#[derive(Debug)]
+pub struct GenericStore<K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    indices: Arc<Indices<K>>,
+
+    region_manager: Arc<RegionManager<D, EP, EL>>,
+
+    device: D,
+
+    admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
+    reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
+
+    flusher_handles: Mutex<Vec<JoinHandle<()>>>,
+    flushers_stop_tx: broadcast::Sender<()>,
+
+    reclaimer_handles: Mutex<Vec<JoinHandle<()>>>,
+    reclaimers_stop_tx: broadcast::Sender<()>,
+
+    metrics: Arc<Metrics>,
+
+    _marker: PhantomData<V>,
+}
+
+impl<K, V, D, EP, EL> GenericStore<K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    async fn open(config: GenericStoreConfig<K, V, D, EP>) -> Result<Arc<Self>> {
+        tracing::info!("open store with config:\n{:#?}", config);
+
+        let metrics = Arc::new(METRICS.foyer(&config.name));
+
+        let device = D::open(config.device_config).await?;
+
+        let buffer_count = config.buffer_pool_size / device.region_size();
+
+        if buffer_count < (1 << config.allocator_bits) {
+            return Err(anyhow::anyhow!(
+                "The count of allocators shoule be greater than buffer count."
+            )
+            .into());
+        }
+
+        let region_manager = Arc::new(RegionManager::new(
+            config.allocator_bits,
+            buffer_count,
+            device.regions(),
+            config.eviction_config,
+            device.clone(),
+            config.allocation_timeout,
+            metrics.clone(),
+        ));
+
+        let indices = Arc::new(Indices::new(device.regions()));
+
+        let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
+        let (reclaimers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
+
+        let flusher_stop_rxs = (0..config.flushers)
+            .map(|_| flushers_stop_tx.subscribe())
+            .collect_vec();
+        let reclaimer_stop_rxs = (0..config.reclaimers)
+            .map(|_| reclaimers_stop_tx.subscribe())
+            .collect_vec();
+
+        let store = Arc::new(Self {
+            indices: indices.clone(),
+            region_manager: region_manager.clone(),
+            device: device.clone(),
+            admissions: config.admissions,
+            reinsertions: config.reinsertions,
+            flusher_handles: Mutex::new(vec![]),
+            reclaimer_handles: Mutex::new(vec![]),
+            flushers_stop_tx,
+            reclaimers_stop_tx,
+            metrics: metrics.clone(),
+            _marker: PhantomData,
+        });
+
+        for admission in store.admissions.iter() {
+            admission.init(&store.indices);
+        }
+        for reinsertion in store.reinsertions.iter() {
+            reinsertion.init(&store.indices);
+        }
+
+        let flush_rate_limiter = match config.flush_rate_limit {
+            0 => None,
+            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
+        };
+        let reclaim_rate_limiter = match config.reclaim_rate_limit {
+            0 => None,
+            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
+        };
+
+        let flushers = flusher_stop_rxs
+            .into_iter()
+            .map(|stop_rx| {
+                Flusher::new(
+                    region_manager.clone(),
+                    flush_rate_limiter.clone(),
+                    metrics.clone(),
+                    stop_rx,
+                )
+            })
+            .collect_vec();
+        let reclaimers = reclaimer_stop_rxs
+            .into_iter()
+            .map(|stop_rx| {
+                Reclaimer::new(
+                    config.clean_region_threshold,
+                    store.clone(),
+                    region_manager.clone(),
+                    reclaim_rate_limiter.clone(),
+                    metrics.clone(),
+                    stop_rx,
+                )
+            })
+            .collect_vec();
+
+        store.recover(config.recover_concurrency).await?;
+
+        let flusher_handles = flushers
+            .into_iter()
+            .map(|flusher| tokio::spawn(async move { flusher.run().await.unwrap() }))
+            .collect_vec();
+
+        let reclaimer_handles = reclaimers
+            .into_iter()
+            .map(|reclaimer| tokio::spawn(async move { reclaimer.run().await.unwrap() }))
+            .collect_vec();
+
+        *store.flusher_handles.lock() = flusher_handles;
+        *store.reclaimer_handles.lock() = reclaimer_handles;
+
+        Ok(store)
+    }
+
+    async fn close(&self) -> Result<()> {
+        // seal current dirty buffer and trigger flushing
+        self.seal().await;
+
+        // stop and wait for reclaimers
+        let handles = self.reclaimer_handles.lock().drain(..).collect_vec();
+        if !handles.is_empty() {
+            self.reclaimers_stop_tx.send(()).unwrap();
+        }
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        // stop and wait for flushers
+        let handles = self.flusher_handles.lock().drain(..).collect_vec();
+        if !handles.is_empty() {
+            self.flushers_stop_tx.send(()).unwrap();
+        }
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        Ok(())
+    }
+
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self))]
+    fn writer(&self, key: K, weight: usize) -> GenericStoreWriter<'_, K, V, D, EP, EL> {
+        GenericStoreWriter::new(self, key, weight)
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn exists(&self, key: &K) -> Result<bool> {
+        Ok(self.indices.lookup(key).is_some())
+    }
+
+    #[tracing::instrument(skip(self))]
+    async fn lookup(&self, key: &K) -> Result<Option<V>> {
+        let now = Instant::now();
+
+        let index = match self.indices.lookup(key) {
+            Some(index) => index,
+            None => {
+                self.metrics
+                    .op_duration_lookup_miss
+                    .observe(now.elapsed().as_secs_f64());
+                return Ok(None);
+            }
+        };
+
+        self.region_manager.record_access(&index.region);
+        let region = self.region_manager.region(&index.region);
+        let start = index.offset as usize;
+        let end = start + index.len as usize;
+
+        // TODO(MrCroxx): read value only
+        let slice = match region.load(start..end, index.version).await? {
+            Some(slice) => slice,
+            None => {
+                // Remove index if the storage layer fails to lookup it (because of region version mismatch).
+                self.indices.remove(key);
+                self.metrics
+                    .op_duration_lookup_miss
+                    .observe(now.elapsed().as_secs_f64());
+                return Ok(None);
+            }
+        };
+        self.metrics.op_bytes_lookup.inc_by(slice.len() as u64);
+
+        let res = match read_entry::<K, V>(slice.as_ref()) {
+            Some((_key, value)) => Ok(Some(value)),
+            None => {
+                // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
+                self.indices.remove(key);
+                Ok(None)
+            }
+        };
+        drop(slice);
+
+        self.metrics
+            .op_duration_lookup_hit
+            .observe(now.elapsed().as_secs_f64());
+
+        res
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn remove(&self, key: &K) -> Result<bool> {
+        let _timer = self.metrics.op_duration_remove.start_timer();
+
+        let res = self.indices.remove(key).is_some();
+
+        Ok(res)
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn clear(&self) -> Result<()> {
+        self.indices.clear();
+
+        // TODO(MrCroxx): set all regions as clean?
+
+        Ok(())
+    }
+
+    pub(crate) fn indices(&self) -> &Arc<Indices<K>> {
+        &self.indices
+    }
+
+    pub(crate) fn reinsertions(&self) -> &Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>> {
+        &self.reinsertions
+    }
+
+    fn serialized_len(&self, key: &K, value: &V) -> usize {
+        let unaligned =
+            EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len();
+        bits::align_up(self.device.align(), unaligned)
+    }
+
+    async fn seal(&self) {
+        self.region_manager.seal().await;
+    }
+
+    #[tracing::instrument(skip(self))]
+    async fn recover(&self, concurrency: usize) -> Result<()> {
+        tracing::info!("start store recovery");
+
+        let (tx, rx) = async_channel::bounded(concurrency);
+
+        let mut handles = vec![];
+        for region_id in 0..self.device.regions() as RegionId {
+            let itx = tx.clone();
+            let irx = rx.clone();
+            let region_manager = self.region_manager.clone();
+            let indices = self.indices.clone();
+            let handle = tokio::spawn(async move {
+                itx.send(()).await.unwrap();
+                let res = Self::recover_region(region_id, region_manager, indices).await;
+                irx.recv().await.unwrap();
+                res
+            });
+            handles.push(handle);
+        }
+
+        let mut recovered = 0;
+
+        let results = try_join_all(handles).await.map_err(anyhow::Error::from)?;
+
+        for (region_id, result) in results.into_iter().enumerate() {
+            if result? {
+                tracing::debug!("region {} is recovered", region_id);
+                recovered += 1
+            }
+        }
+
+        tracing::info!("finish store recovery, {} region recovered", recovered);
+        self.metrics
+            .total_bytes
+            .set((recovered * self.device.region_size()) as u64);
+
+        // Force trigger reclamation.
+        if recovered == self.device.regions() {
+            self.region_manager.clean_regions().flash();
+        }
+
+        Ok(())
+    }
+
+    /// Return `true` if region is valid, otherwise `false`
+    async fn recover_region(
+        region_id: RegionId,
+        region_manager: Arc<RegionManager<D, EP, EL>>,
+        indices: Arc<Indices<K>>,
+    ) -> Result<bool> {
+        let region = region_manager.region(&region_id).clone();
+        let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
+            while let Some(index) = iter.next().await? {
+                indices.insert(index);
+            }
+            region_manager.eviction_push(region_id);
+            true
+        } else {
+            region_manager.clean_regions().release(region_id);
+            false
+        };
+        Ok(res)
+    }
+
+    fn judge_inner(&self, writer: &mut GenericStoreWriter<'_, K, V, D, EP, EL>) {
+        for (index, admission) in self.admissions.iter().enumerate() {
+            let judge = admission.judge(&writer.key, writer.weight, &self.metrics);
+            writer.judges.set(index, judge);
+        }
+        writer.is_judged = true;
+    }
+
+    async fn apply_writer(
+        &self,
+        mut writer: GenericStoreWriter<'_, K, V, D, EP, EL>,
+        value: V,
+    ) -> Result<bool> {
+        debug_assert!(!writer.is_inserted);
+
+        if !writer.judge() {
+            return Ok(false);
+        }
+
+        let now = Instant::now();
+
+        writer.is_inserted = true;
+        let key = &writer.key;
+
+        for (i, admission) in self.admissions.iter().enumerate() {
+            let judge = writer.judges.get(i);
+            admission.on_insert(key, writer.weight, &self.metrics, judge);
+        }
+
+        let serialized_len = self.serialized_len(key, &value);
+
+        if key.serialized_len() + value.serialized_len() != writer.weight {
+            tracing::error!(
+                "weight != key.serialized_len() + value.serialized_len(), weight: {}, key size: {}, value size: {}, key: {:?}",
+                writer.weight, key.serialized_len(), value.serialized_len(), key
+            );
+        }
+
+        self.metrics.op_bytes_insert.inc_by(serialized_len as u64);
+
+        let mut slice = match self
+            .region_manager
+            .allocate(serialized_len, !writer.is_skippable)
+            .await
+        {
+            Some(slice) => slice,
+            // Only reachable when writer is skippable.
+            None => return Ok(false),
+        };
+
+        write_entry(slice.as_mut(), key, &value);
+
+        let index = Index {
+            region: slice.region_id(),
+            version: slice.version(),
+            offset: slice.offset() as u32,
+            len: slice.len() as u32,
+            key_len: key.serialized_len() as u32,
+            value_len: value.serialized_len() as u32,
+
+            key: key.clone(),
+        };
+        drop(slice);
+
+        self.indices.insert(index);
+
+        let duration = now.elapsed() + writer.duration;
+        self.metrics
+            .op_duration_insert_inserted
+            .observe(duration.as_secs_f64());
+
+        Ok(true)
+    }
+}
+
+pub struct GenericStoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    store: &'a GenericStore<K, V, D, EP, EL>,
+    key: K,
+    weight: usize,
+
+    judges: Judges,
+    is_judged: bool,
+
+    /// judge duration
+    duration: Duration,
+
+    is_inserted: bool,
+    is_skippable: bool,
+}
+
+impl<'a, K, V, D, EP, EL> GenericStoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    fn new(store: &'a GenericStore<K, V, D, EP, EL>, key: K, weight: usize) -> Self {
+        Self {
+            store,
+            key,
+            weight,
+            judges: Judges::new(store.admissions.len()),
+            is_judged: false,
+            duration: Duration::from_nanos(0),
+            is_inserted: false,
+            is_skippable: false,
+        }
+    }
+
+    /// Judge if the entry can be admitted by configured admission policies.
+    pub fn judge(&mut self) -> bool {
+        if !self.is_judged {
+            let now = Instant::now();
+            self.store.judge_inner(self);
+            self.duration = now.elapsed();
+        }
+        self.judges.judge()
+    }
+
+    pub async fn finish(self, value: V) -> Result<bool> {
+        self.store.apply_writer(self, value).await
+    }
+
+    pub fn set_force(&mut self) {
+        self.judges.set_mask(Bitmap::new());
+    }
+
+    pub fn set_judge_mask(&mut self, mask: Bitmap<64>) {
+        self.judges.set_mask(mask);
+    }
+
+    pub fn set_skippable(&mut self) {
+        self.is_skippable = true
+    }
+}
+
+impl<'a, K, V, D, EP, EL> Debug for GenericStoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("StoreWriter")
+            .field("key", &self.key)
+            .field("weight", &self.weight)
+            .field("judges", &self.judges)
+            .field("is_judged", &self.is_judged)
+            .field("duration", &self.duration)
+            .field("inserted", &self.is_inserted)
+            .finish()
+    }
+}
+
+impl<'a, K, V, D, EP, EL> Drop for GenericStoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    fn drop(&mut self) {
+        if !self.is_inserted {
+            self.store
+                .metrics
+                .op_duration_insert_dropped
+                .observe(self.duration.as_secs_f64());
+            let mut filtered = false;
+            if self.is_judged {
+                for (i, admission) in self.store.admissions.iter().enumerate() {
+                    let judge = self.judges.get(i);
+                    admission.on_drop(&self.key, self.weight, &self.store.metrics, judge);
+                }
+                filtered = !self.judge();
+            }
+            if filtered {
+                self.store
+                    .metrics
+                    .op_duration_insert_filtered
+                    .observe(self.duration.as_secs_f64());
+            } else {
+                self.store
+                    .metrics
+                    .op_duration_insert_dropped
+                    .observe(self.duration.as_secs_f64());
+            }
+        }
+    }
+}
+
+const ENTRY_MAGIC: u32 = 0x97_00_00_00;
+const ENTRY_MAGIC_MASK: u32 = 0xFF_00_00_00;
+
+#[derive(Debug)]
+struct EntryHeader {
+    key_len: u32,
+    value_len: u32,
+    checksum: u64,
+}
+
+impl EntryHeader {
+    fn serialized_len() -> usize {
+        4 + 4 + 8
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        buf.put_u32(self.key_len | ENTRY_MAGIC);
+        buf.put_u32(self.value_len);
+        buf.put_u64(self.checksum);
+    }
+
+    fn read(mut buf: &[u8]) -> Option<Self> {
+        let head = buf.get_u32();
+        let magic = head & ENTRY_MAGIC_MASK;
+
+        if magic != ENTRY_MAGIC {
+            return None;
+        }
+
+        let key_len = head ^ ENTRY_MAGIC;
+        let value_len = buf.get_u32();
+        let checksum = buf.get_u64();
+
+        Some(Self {
+            key_len,
+            value_len,
+            checksum,
+        })
+    }
+}
+
+/// | header | value | key | <padding> |
+///
+/// # Safety
+///
+/// `buf.len()` must excatly fit entry size
+fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V)
+where
+    K: Key,
+    V: Value,
+{
+    let mut offset = EntryHeader::serialized_len();
+    value.write(&mut buf[offset..offset + value.serialized_len()]);
+    offset += value.serialized_len();
+    key.write(&mut buf[offset..offset + key.serialized_len()]);
+    offset += key.serialized_len();
+    let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
+
+    let header = EntryHeader {
+        key_len: key.serialized_len() as u32,
+        value_len: value.serialized_len() as u32,
+        checksum,
+    };
+    header.write(&mut buf[..EntryHeader::serialized_len()]);
+}
+
+/// | header | value | key | <padding> |
+///
+/// # Safety
+///
+/// `buf.len()` must excatly fit entry size
+fn read_entry<K, V>(buf: &[u8]) -> Option<(K, V)>
+where
+    K: Key,
+    V: Value,
+{
+    let header = EntryHeader::read(buf)?;
+
+    let mut offset = EntryHeader::serialized_len();
+    let value = V::read(&buf[offset..offset + header.value_len as usize]);
+    offset += header.value_len as usize;
+    let key = K::read(&buf[offset..offset + header.key_len as usize]);
+    offset += header.key_len as usize;
+
+    let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
+    if checksum != header.checksum {
+        tracing::warn!(
+            "checksum mismatch, checksum: {}, expected: {}",
+            checksum,
+            header.checksum,
+        );
+        return None;
+    }
+
+    Some((key, value))
+}
+
+fn checksum(buf: &[u8]) -> u64 {
+    let mut hasher = XxHash64::with_seed(0);
+    hasher.write(buf);
+    hasher.finish()
+}
+
+pub struct RegionEntryIter<K, V, D>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+{
+    region: Region<D>,
+
+    cursor: usize,
+
+    _marker: PhantomData<(K, V)>,
+}
+
+impl<K, V, D> RegionEntryIter<K, V, D>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+{
+    pub async fn open(region: Region<D>) -> Result<Option<Self>> {
+        let align = region.device().align();
+
+        let slice = match region.load(..align, 0).await? {
+            Some(slice) => slice,
+            None => return Ok(None),
+        };
+
+        let header = RegionHeader::read(slice.as_ref());
+        drop(slice);
+
+        if header.magic != REGION_MAGIC {
+            return Ok(None);
+        }
+
+        Ok(Some(Self {
+            region,
+            cursor: align,
+            _marker: PhantomData,
+        }))
+    }
+
+    pub async fn next(&mut self) -> Result<Option<Index<K>>> {
+        let region_size = self.region.device().region_size();
+        let align = self.region.device().align();
+
+        if self.cursor + align >= region_size {
+            return Ok(None);
+        }
+
+        let Some(slice) = self
+            .region
+            .load(self.cursor..self.cursor + align, 0)
+            .await?
+        else {
+            return Ok(None);
+        };
+
+        let Some(header) = EntryHeader::read(slice.as_ref()) else {
+            return Ok(None);
+        };
+
+        let entry_len = bits::align_up(
+            align,
+            (header.value_len + header.key_len) as usize + EntryHeader::serialized_len(),
+        );
+
+        let abs_start = self.cursor + EntryHeader::serialized_len() + header.value_len as usize;
+        let abs_end = self.cursor
+            + EntryHeader::serialized_len()
+            + (header.key_len + header.value_len) as usize;
+
+        if abs_start >= abs_end || abs_end > region_size {
+            // Double check wrong entry.
+            return Ok(None);
+        }
+
+        let align_start = bits::align_down(align, abs_start);
+        let align_end = bits::align_up(align, abs_end);
+
+        let key = if align_start == self.cursor - align && align_end == self.cursor {
+            // header and key are in the same block, read directly from slice
+            let rel_start = EntryHeader::serialized_len() + header.value_len as usize;
+            let rel_end = rel_start + header.key_len as usize;
+            let key = K::read(&slice.as_ref()[rel_start..rel_end]);
+            drop(slice);
+            key
+        } else {
+            drop(slice);
+            let Some(s) = self.region.load(align_start..align_end, 0).await? else {
+                return Ok(None);
+            };
+            let rel_start = abs_start - align_start;
+            let rel_end = abs_end - align_start;
+
+            let key = K::read(&s.as_ref()[rel_start..rel_end]);
+            drop(s);
+            key
+        };
+
+        let index = Index {
+            key,
+            region: self.region.id(),
+            version: 0,
+            offset: self.cursor as u32,
+            len: entry_len as u32,
+            key_len: header.key_len,
+            value_len: header.value_len,
+        };
+
+        self.cursor += entry_len;
+
+        Ok(Some(index))
+    }
+
+    pub async fn next_kv(&mut self) -> Result<Option<(K, V)>> {
+        let index = match self.next().await {
+            Ok(Some(index)) => index,
+            Ok(None) => return Ok(None),
+            Err(e) => return Err(e),
+        };
+
+        // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
+        let start = index.offset as usize;
+        let end = start + index.len as usize;
+        let Some(slice) = self.region.load(start..end, 0).await? else {
+            return Ok(None);
+        };
+        let kv = read_entry::<K, V>(slice.as_ref());
+        drop(slice);
+
+        Ok(kv)
+    }
+}
+
+impl<'a, K, V, D, EP, EL> StorageWriter for GenericStoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    type Key = K;
+    type Value = V;
+
+    fn judge(&mut self) -> bool {
+        self.judge()
+    }
+
+    async fn finish(self, value: Self::Value) -> Result<bool> {
+        self.finish(value).await
+    }
+}
+
+impl<'a, K, V, D, EP, EL> ForceStorageWriter for GenericStoreWriter<'a, K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    fn set_force(&mut self) {
+        self.set_force()
+    }
+}
+
+impl<K, V, D, EP, EL> Storage for GenericStore<K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    type Key = K;
+    type Value = V;
+    type Config = GenericStoreConfig<K, V, D, EP>;
+    type Owned = Arc<Self>;
+    type Writer<'a> = GenericStoreWriter<'a, K, V, D, EP, EL>;
+
+    async fn open(config: Self::Config) -> Result<Self::Owned> {
+        Self::open(config).await
+    }
+
+    async fn close(&self) -> Result<()> {
+        self.close().await
+    }
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+        self.writer(key, weight)
+    }
+
+    fn exists(&self, key: &Self::Key) -> Result<bool> {
+        self.exists(key)
+    }
+
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        self.lookup(key).await
+    }
+
+    fn remove(&self, key: &Self::Key) -> Result<bool> {
+        self.remove(key)
+    }
+
+    fn clear(&self) -> Result<()> {
+        self.clear()
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use std::{collections::HashSet, path::PathBuf};
+
+    use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
+
+    use crate::{
+        device::fs::{FsDevice, FsDeviceConfig},
+        storage::StorageExt,
+    };
+
+    use super::*;
+
+    type TestStore =
+        GenericStore<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
+
+    type TestStoreConfig =
+        GenericStoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
+
+    #[derive(Debug, Clone)]
+    enum Record<K: Key> {
+        Admit(K),
+        Evict(K),
+    }
+
+    #[derive(Debug)]
+    struct JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        records: Mutex<Vec<Record<K>>>,
+        _marker: PhantomData<V>,
+    }
+
+    impl<K, V> JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        fn dump(&self) -> Vec<Record<K>> {
+            self.records.lock().clone()
+        }
+
+        fn remains(&self) -> HashSet<K> {
+            let records = self.dump();
+            let mut res = HashSet::default();
+            for record in records {
+                match record {
+                    Record::Admit(key) => {
+                        res.insert(key);
+                    }
+                    Record::Evict(key) => {
+                        res.remove(&key);
+                    }
+                }
+            }
+            res
+        }
+    }
+
+    impl<K, V> Default for JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        fn default() -> Self {
+            Self {
+                records: Mutex::new(Vec::default()),
+                _marker: PhantomData,
+            }
+        }
+    }
+
+    impl<K, V> AdmissionPolicy for JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        type Key = K;
+
+        type Value = V;
+
+        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+            self.records.lock().push(Record::Admit(key.clone()));
+            true
+        }
+
+        fn on_insert(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+
+        fn on_drop(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+    }
+
+    impl<K, V> ReinsertionPolicy for JudgeRecorder<K, V>
+    where
+        K: Key,
+        V: Value,
+    {
+        type Key = K;
+
+        type Value = V;
+
+        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+            self.records.lock().push(Record::Evict(key.clone()));
+            false
+        }
+
+        fn on_insert(
+            &self,
+            _key: &Self::Key,
+            _weight: usize,
+            _metrics: &Arc<crate::metrics::Metrics>,
+            _judge: bool,
+        ) {
+        }
+
+        fn on_drop(
+            &self,
+            _key: &Self::Key,
+            _weight: usize,
+            _metrics: &Arc<crate::metrics::Metrics>,
+            _judge: bool,
+        ) {
+        }
+    }
+
+    #[tokio::test]
+    #[expect(clippy::identity_op)]
+    async fn test_recovery() {
+        const KB: usize = 1024;
+        const MB: usize = 1024 * 1024;
+
+        let tempdir = tempfile::tempdir().unwrap();
+
+        let recorder = Arc::new(JudgeRecorder::default());
+        let admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> =
+            vec![recorder.clone()];
+        let reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> =
+            vec![recorder.clone()];
+
+        let config = TestStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            allocator_bits: 1,
+            admissions,
+            reinsertions,
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 1,
+            reclaim_rate_limit: 0,
+            recover_concurrency: 2,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+        };
+
+        let store = TestStore::open(config).await.unwrap();
+
+        // files:
+        // [0, 1, 2]
+        // [3, 4, 5]
+        // [6, 7, 8]
+        // [9, 10, 11]
+        for i in 0..20 {
+            store.insert(i, vec![i as u8; 1 * MB]).await.unwrap();
+        }
+
+        store.close().await.unwrap();
+
+        let remains = recorder.remains();
+
+        for i in 0..20 {
+            if remains.contains(&i) {
+                assert_eq!(
+                    store.lookup(&i).await.unwrap().unwrap(),
+                    vec![i as u8; 1 * MB],
+                );
+            } else {
+                assert!(store.lookup(&i).await.unwrap().is_none());
+            }
+        }
+
+        drop(store);
+
+        let config = TestStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            allocator_bits: 1,
+            admissions: vec![],
+            reinsertions: vec![],
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 0,
+            reclaim_rate_limit: 0,
+            recover_concurrency: 2,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+        };
+        let store = TestStore::open(config).await.unwrap();
+
+        for i in 0..12 {
+            if remains.contains(&i) {
+                assert_eq!(
+                    store.lookup(&i).await.unwrap().unwrap(),
+                    vec![i as u8; 1 * MB],
+                );
+            } else {
+                assert!(store.lookup(&i).await.unwrap().is_none());
+            }
+        }
+
+        store.close().await.unwrap();
+
+        drop(store);
+    }
+}
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
new file mode 100644
index 00000000..7da1803f
--- /dev/null
+++ b/foyer-storage/src/lazy.rs
@@ -0,0 +1,222 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::{Arc, OnceLock};
+
+use crate::{
+    error::Result,
+    storage::Storage,
+    store::{Store, StoreConfig, StoreWriter},
+};
+use foyer_common::code::{Key, Value};
+use futures::Future;
+
+#[derive(Debug, Clone)]
+pub struct LazyStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    once: Arc<OnceLock<Store<K, V>>>,
+    none: Store<K, V>,
+}
+
+impl<K, V> LazyStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn lazy(config: StoreConfig<K, V>) -> Self {
+        let (res, task) = Self::lazy_with_task(config);
+        tokio::spawn(task);
+        res
+    }
+
+    pub fn lazy_with_task(
+        config: StoreConfig<K, V>,
+    ) -> (Self, impl Future<Output = Result<Store<K, V>>> + Send) {
+        let once = Arc::new(OnceLock::new());
+
+        let task = {
+            let once = once.clone();
+            async move {
+                let store = match Store::open(config).await {
+                    Ok(store) => store,
+                    Err(e) => {
+                        tracing::error!("Lazy open store fail: {}", e);
+                        return Err(e);
+                    }
+                };
+                once.set(store.clone()).unwrap();
+                Ok(store)
+            }
+        };
+
+        let res = Self {
+            once,
+            none: Store::None,
+        };
+
+        (res, task)
+    }
+}
+
+impl<K, V> Storage for LazyStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Config = StoreConfig<K, V>;
+    type Owned = Self;
+    type Writer<'a> = StoreWriter<'a, K, V>;
+
+    async fn open(config: Self::Config) -> Result<Self::Owned> {
+        let once = Arc::new(OnceLock::new());
+        let store = Store::open(config).await?;
+        once.set(store).unwrap();
+        Ok(Self {
+            once,
+            none: Store::None,
+        })
+    }
+
+    async fn close(&self) -> Result<()> {
+        match self.once.get() {
+            Some(store) => store.close().await,
+            None => self.none.close().await,
+        }
+    }
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+        match self.once.get() {
+            Some(store) => store.writer(key, weight),
+            None => self.none.writer(key, weight),
+        }
+    }
+
+    fn exists(&self, key: &Self::Key) -> Result<bool> {
+        match self.once.get() {
+            Some(store) => store.exists(key),
+            None => self.none.exists(key),
+        }
+    }
+
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        match self.once.get() {
+            Some(store) => store.lookup(key).await,
+            None => self.none.lookup(key).await,
+        }
+    }
+
+    fn remove(&self, key: &Self::Key) -> Result<bool> {
+        match self.once.get() {
+            Some(store) => store.remove(key),
+            None => self.none.remove(key),
+        }
+    }
+
+    fn clear(&self) -> Result<()> {
+        match self.once.get() {
+            Some(store) => store.clear(),
+            None => self.none.clear(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{path::PathBuf, time::Duration};
+
+    use foyer_intrusive::eviction::fifo::FifoConfig;
+
+    use crate::{device::fs::FsDeviceConfig, storage::StorageExt, store::FifoFsStoreConfig};
+
+    use super::*;
+
+    const KB: usize = 1024;
+    const MB: usize = 1024 * 1024;
+
+    #[tokio::test]
+    async fn test_lazy_store() {
+        let tempdir = tempfile::tempdir().unwrap();
+
+        let config = FifoFsStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            allocator_bits: 1,
+            admissions: vec![],
+            reinsertions: vec![],
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 1,
+            reclaim_rate_limit: 0,
+            recover_concurrency: 2,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+        };
+
+        let (store, task) = LazyStore::lazy_with_task(config.into());
+
+        assert!(!store.insert(100, 100).await.unwrap());
+
+        tokio::spawn(task).await.unwrap().unwrap();
+
+        assert!(store.insert(100, 100).await.unwrap());
+        assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
+
+        store.close().await.unwrap();
+        drop(store);
+
+        let config = FifoFsStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 16 * MB,
+                file_capacity: 4 * MB,
+                align: 4096,
+                io_size: 4096 * KB,
+            },
+            allocator_bits: 1,
+            admissions: vec![],
+            reinsertions: vec![],
+            buffer_pool_size: 8 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 1,
+            reclaim_rate_limit: 0,
+            recover_concurrency: 2,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+        };
+
+        let (store, task) = LazyStore::lazy_with_task(config.into());
+
+        assert!(store.lookup(&100).await.unwrap().is_none());
+
+        tokio::spawn(task).await.unwrap().unwrap();
+
+        assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
+    }
+}
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 40460247..54fcd6f9 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -24,826 +24,19 @@
 #![feature(return_position_impl_trait_in_trait)]
 #![feature(associated_type_defaults)]
 
-use std::{
-    fmt::Debug,
-    sync::{Arc, OnceLock},
-};
-
-use foyer_common::code::{Key, Value};
-
-use error::Result;
-use futures::Future;
-
 pub mod admission;
 pub mod device;
 pub mod error;
 pub mod flusher;
+pub mod generic;
 pub mod indices;
 pub mod judge;
+pub mod lazy;
 pub mod metrics;
 pub mod reclaimer;
 pub mod region;
 pub mod region_manager;
 pub mod reinsertion;
 pub mod slice;
+pub mod storage;
 pub mod store;
-
-pub type LruFsStore<K, V> = store::Store<
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::lru::Lru<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
-    >,
-    foyer_intrusive::eviction::lru::LruLink,
->;
-
-pub type LruFsStoreConfig<K, V> = store::StoreConfig<
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::lru::Lru<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
-    >,
->;
-
-pub type LruFsStoreWriter<'w, K, V> = store::StoreWriter<
-    'w,
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::lru::Lru<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lru::LruLink>,
-    >,
-    foyer_intrusive::eviction::lru::LruLink,
->;
-
-pub type LfuFsStore<K, V> = store::Store<
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::lfu::Lfu<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
-    >,
-    foyer_intrusive::eviction::lfu::LfuLink,
->;
-
-pub type LfuFsStoreConfig<K, V> = store::StoreConfig<
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::lfu::Lfu<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
-    >,
->;
-
-pub type LfuFsStoreWriter<'w, K, V> = store::StoreWriter<
-    'w,
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::lfu::Lfu<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::lfu::LfuLink>,
-    >,
-    foyer_intrusive::eviction::lfu::LfuLink,
->;
-
-pub type FifoFsStore<K, V> = store::Store<
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::fifo::Fifo<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
-    >,
-    foyer_intrusive::eviction::fifo::FifoLink,
->;
-
-pub type FifoFsStoreConfig<K, V> = store::StoreConfig<
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::fifo::Fifo<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
-    >,
->;
-
-pub type FifoFsStoreWriter<'w, K, V> = store::StoreWriter<
-    'w,
-    K,
-    V,
-    device::fs::FsDevice,
-    foyer_intrusive::eviction::fifo::Fifo<
-        region_manager::RegionEpItemAdapter<foyer_intrusive::eviction::fifo::FifoLink>,
-    >,
-    foyer_intrusive::eviction::fifo::FifoLink,
->;
-
-pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
-
-pub trait StorageWriter: Send + Sync + Debug {
-    type Key: Key;
-    type Value: Value;
-
-    fn judge(&mut self) -> bool;
-
-    fn finish(self, value: Self::Value) -> impl Future<Output = Result<bool>> + Send;
-}
-
-pub trait Storage: Send + Sync + Debug + 'static {
-    type Key: Key;
-    type Value: Value;
-    type Config: Send + Debug;
-    type Owned: Send + Sync + Debug + 'static;
-    type Writer<'a>: StorageWriter<Key = Self::Key, Value = Self::Value>;
-
-    #[must_use]
-    fn open(config: Self::Config) -> impl Future<Output = Result<Self::Owned>> + Send;
-
-    #[must_use]
-    fn close(&self) -> impl Future<Output = Result<()>> + Send;
-
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_>;
-
-    fn exists(&self, key: &Self::Key) -> Result<bool>;
-
-    #[must_use]
-    fn lookup(&self, key: &Self::Key) -> impl Future<Output = Result<Option<Self::Value>>> + Send;
-
-    fn remove(&self, key: &Self::Key) -> Result<bool>;
-
-    fn clear(&self) -> Result<()>;
-}
-
-pub trait StorageExt: Storage {
-    #[must_use]
-    #[tracing::instrument(skip(self, value))]
-    fn insert(
-        &self,
-        key: Self::Key,
-        value: Self::Value,
-    ) -> impl Future<Output = Result<bool>> + Send {
-        let weight = key.serialized_len() + value.serialized_len();
-        self.writer(key, weight).finish(value)
-    }
-
-    #[must_use]
-    #[tracing::instrument(skip(self, value))]
-    fn insert_if_not_exists(
-        &self,
-        key: Self::Key,
-        value: Self::Value,
-    ) -> impl Future<Output = Result<bool>> + Send {
-        async move {
-            if self.exists(&key)? {
-                return Ok(false);
-            }
-            self.insert(key, value).await
-        }
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[must_use]
-    #[tracing::instrument(skip(self, f))]
-    fn insert_with<F>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
-    where
-        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
-    {
-        async move {
-            let mut writer = self.writer(key, weight);
-            if !writer.judge() {
-                return Ok(false);
-            }
-            let value = match f() {
-                Ok(value) => value,
-                Err(e) => {
-                    tracing::warn!("fetch value error: {:?}", e);
-                    return Ok(false);
-                }
-            };
-            writer.finish(value).await
-        }
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called to fetch value, and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self, f))]
-    fn insert_with_future<F, FU>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
-    where
-        F: FnOnce() -> FU + Send,
-        FU: FetchValueFuture<Self::Value>,
-    {
-        async move {
-            let mut writer = self.writer(key, weight);
-            if !writer.judge() {
-                return Ok(false);
-            }
-            let value = match f().await {
-                Ok(value) => value,
-                Err(e) => {
-                    tracing::warn!("fetch value error: {:?}", e);
-                    return Ok(false);
-                }
-            };
-            writer.finish(value).await
-        }
-    }
-
-    #[tracing::instrument(skip(self, f))]
-    fn insert_if_not_exists_with<F>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
-    where
-        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
-    {
-        async move {
-            if self.exists(&key)? {
-                return Ok(false);
-            }
-            self.insert_with(key, f, weight).await
-        }
-    }
-
-    #[tracing::instrument(skip(self, f))]
-    fn insert_if_not_exists_with_future<F, FU>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
-    where
-        F: FnOnce() -> FU + Send,
-        FU: FetchValueFuture<Self::Value>,
-    {
-        async move {
-            if self.exists(&key)? {
-                return Ok(false);
-            }
-            self.insert_with_future(key, f, weight).await
-        }
-    }
-}
-
-impl<S: Storage> StorageExt for S {}
-
-pub trait ForceStorageWriter: StorageWriter {
-    fn set_force(&mut self);
-}
-
-pub trait ForceStorageExt: Storage
-where
-    for<'w> Self::Writer<'w>: ForceStorageWriter,
-{
-    #[tracing::instrument(skip(self, value))]
-    fn insert_force(
-        &self,
-        key: Self::Key,
-        value: Self::Value,
-    ) -> impl Future<Output = Result<bool>> + Send {
-        let weight = key.serialized_len() + value.serialized_len();
-        let mut writer = self.writer(key, weight);
-        writer.set_force();
-        writer.finish(value)
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self, f))]
-    fn insert_force_with<F>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
-    where
-        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
-    {
-        async move {
-            let mut writer = self.writer(key, weight);
-            writer.set_force();
-            if !writer.judge() {
-                return Ok(false);
-            }
-            let value = match f() {
-                Ok(value) => value,
-                Err(e) => {
-                    tracing::warn!("fetch value error: {:?}", e);
-                    return Ok(false);
-                }
-            };
-            let inserted = writer.finish(value).await?;
-            Ok(inserted)
-        }
-    }
-
-    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
-    /// Then `f` will be called to fetch value, and entry will be inserted.
-    ///
-    /// # Safety
-    ///
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self, f))]
-    fn insert_force_with_future<F, FU>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
-    where
-        F: FnOnce() -> FU + Send,
-        FU: FetchValueFuture<Self::Value>,
-    {
-        async move {
-            let mut writer = self.writer(key, weight);
-            writer.set_force();
-            if !writer.judge() {
-                return Ok(false);
-            }
-            let value = match f().await {
-                Ok(value) => value,
-                Err(e) => {
-                    tracing::warn!("fetch value error: {:?}", e);
-                    return Ok(false);
-                }
-            };
-            let inserted = writer.finish(value).await?;
-            Ok(inserted)
-        }
-    }
-}
-
-impl<S> ForceStorageExt for S
-where
-    S: Storage,
-    for<'w> S::Writer<'w>: ForceStorageWriter,
-{
-}
-
-#[derive(Debug)]
-pub enum StoreConfig<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    LruFsStoreConfig { config: LruFsStoreConfig<K, V> },
-    LfuFsStoreConfig { config: LfuFsStoreConfig<K, V> },
-    FifoFsStoreConfig { config: FifoFsStoreConfig<K, V> },
-    None,
-}
-
-impl<K, V> From<LruFsStoreConfig<K, V>> for StoreConfig<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(config: LruFsStoreConfig<K, V>) -> Self {
-        StoreConfig::LruFsStoreConfig { config }
-    }
-}
-
-impl<K, V> From<LfuFsStoreConfig<K, V>> for StoreConfig<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(config: LfuFsStoreConfig<K, V>) -> Self {
-        StoreConfig::LfuFsStoreConfig { config }
-    }
-}
-
-impl<K, V> From<FifoFsStoreConfig<K, V>> for StoreConfig<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(config: FifoFsStoreConfig<K, V>) -> Self {
-        StoreConfig::FifoFsStoreConfig { config }
-    }
-}
-
-#[derive(Debug)]
-pub enum StoreWriter<'a, K, V>
-where
-    K: Key,
-    V: Value,
-{
-    LruFsStorWriter { writer: LruFsStoreWriter<'a, K, V> },
-    LfuFsStorWriter { writer: LfuFsStoreWriter<'a, K, V> },
-    FifoFsStoreWriter { writer: FifoFsStoreWriter<'a, K, V> },
-    None,
-}
-
-impl<'a, K, V> From<LruFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(writer: LruFsStoreWriter<'a, K, V>) -> Self {
-        StoreWriter::LruFsStorWriter { writer }
-    }
-}
-
-impl<'a, K, V> From<LfuFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(writer: LfuFsStoreWriter<'a, K, V>) -> Self {
-        StoreWriter::LfuFsStorWriter { writer }
-    }
-}
-
-impl<'a, K, V> From<FifoFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(writer: FifoFsStoreWriter<'a, K, V>) -> Self {
-        StoreWriter::FifoFsStoreWriter { writer }
-    }
-}
-
-#[derive(Debug)]
-pub enum Store<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    LruFsStore { store: Arc<LruFsStore<K, V>> },
-    LfuFsStore { store: Arc<LfuFsStore<K, V>> },
-    FifoFsStore { store: Arc<FifoFsStore<K, V>> },
-    None,
-}
-
-impl<K, V> Clone for Store<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn clone(&self) -> Self {
-        match self {
-            Self::LruFsStore { store } => Self::LruFsStore {
-                store: Arc::clone(store),
-            },
-            Self::LfuFsStore { store } => Self::LfuFsStore {
-                store: Arc::clone(store),
-            },
-            Self::FifoFsStore { store } => Self::FifoFsStore {
-                store: Arc::clone(store),
-            },
-            Self::None => Self::None,
-        }
-    }
-}
-
-impl<'a, K, V> StorageWriter for StoreWriter<'a, K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Key = K;
-    type Value = V;
-
-    fn judge(&mut self) -> bool {
-        match self {
-            StoreWriter::LruFsStorWriter { writer } => writer.judge(),
-            StoreWriter::LfuFsStorWriter { writer } => writer.judge(),
-            StoreWriter::FifoFsStoreWriter { writer } => writer.judge(),
-            StoreWriter::None => false,
-        }
-    }
-
-    async fn finish(self, value: Self::Value) -> Result<bool> {
-        match self {
-            StoreWriter::LruFsStorWriter { writer } => writer.finish(value).await,
-            StoreWriter::LfuFsStorWriter { writer } => writer.finish(value).await,
-            StoreWriter::FifoFsStoreWriter { writer } => writer.finish(value).await,
-            StoreWriter::None => Ok(false),
-        }
-    }
-}
-
-impl<'a, K, V> ForceStorageWriter for StoreWriter<'a, K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn set_force(&mut self) {
-        match self {
-            StoreWriter::LruFsStorWriter { writer } => writer.set_force(),
-            StoreWriter::LfuFsStorWriter { writer } => writer.set_force(),
-            StoreWriter::FifoFsStoreWriter { writer } => writer.set_force(),
-            StoreWriter::None => {}
-        }
-    }
-}
-
-impl<K, V> Storage for Store<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Key = K;
-    type Value = V;
-    type Config = StoreConfig<K, V>;
-    type Owned = Self;
-    type Writer<'a> = StoreWriter<'a, K, V>;
-
-    async fn open(config: Self::Config) -> Result<Self::Owned> {
-        match config {
-            StoreConfig::LruFsStoreConfig { config } => {
-                let store = LruFsStore::open(config).await?;
-                Ok(Self::LruFsStore { store })
-            }
-            StoreConfig::LfuFsStoreConfig { config } => {
-                let store = LfuFsStore::open(config).await?;
-                Ok(Self::LfuFsStore { store })
-            }
-            StoreConfig::FifoFsStoreConfig { config } => {
-                let store = FifoFsStore::open(config).await?;
-                Ok(Self::FifoFsStore { store })
-            }
-            StoreConfig::None => Ok(Self::None),
-        }
-    }
-
-    async fn close(&self) -> Result<()> {
-        match self {
-            Store::LruFsStore { store } => store.close().await,
-            Store::LfuFsStore { store } => store.close().await,
-            Store::FifoFsStore { store } => store.close().await,
-            Store::None => Ok(()),
-        }
-    }
-
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
-        match self {
-            Store::LruFsStore { store } => store.writer(key, weight).into(),
-            Store::LfuFsStore { store } => store.writer(key, weight).into(),
-            Store::FifoFsStore { store } => store.writer(key, weight).into(),
-            Store::None => StoreWriter::None,
-        }
-    }
-
-    fn exists(&self, key: &Self::Key) -> Result<bool> {
-        match self {
-            Store::LruFsStore { store } => store.exists(key),
-            Store::LfuFsStore { store } => store.exists(key),
-            Store::FifoFsStore { store } => store.exists(key),
-            Store::None => Ok(false),
-        }
-    }
-
-    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
-        match self {
-            Store::LruFsStore { store } => store.lookup(key).await,
-            Store::LfuFsStore { store } => store.lookup(key).await,
-            Store::FifoFsStore { store } => store.lookup(key).await,
-            Store::None => Ok(None),
-        }
-    }
-
-    fn remove(&self, key: &Self::Key) -> Result<bool> {
-        match self {
-            Store::LruFsStore { store } => store.remove(key),
-            Store::LfuFsStore { store } => store.remove(key),
-            Store::FifoFsStore { store } => store.remove(key),
-            Store::None => Ok(false),
-        }
-    }
-
-    fn clear(&self) -> Result<()> {
-        match self {
-            Store::LruFsStore { store } => store.clear(),
-            Store::LfuFsStore { store } => store.clear(),
-            Store::FifoFsStore { store } => store.clear(),
-            Store::None => Ok(()),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct LazyStore<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    once: Arc<OnceLock<Store<K, V>>>,
-    none: Store<K, V>,
-}
-
-impl<K, V> LazyStore<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    pub fn lazy(config: StoreConfig<K, V>) -> Self {
-        let (res, task) = Self::lazy_with_task(config);
-
-        tokio::spawn(task);
-
-        res
-    }
-
-    pub fn lazy_with_task(
-        config: StoreConfig<K, V>,
-    ) -> (Self, impl Future<Output = Result<Store<K, V>>> + Send) {
-        let once = Arc::new(OnceLock::new());
-
-        let task = {
-            let once = once.clone();
-            async move {
-                let store = match Store::open(config).await {
-                    Ok(store) => store,
-                    Err(e) => {
-                        tracing::error!("Lazy open store fail: {}", e);
-                        return Err(e);
-                    }
-                };
-                once.set(store.clone()).unwrap();
-                Ok(store)
-            }
-        };
-
-        let res = Self {
-            once,
-            none: Store::None,
-        };
-
-        (res, task)
-    }
-}
-
-impl<K, V> Storage for LazyStore<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Key = K;
-    type Value = V;
-    type Config = StoreConfig<K, V>;
-    type Owned = Self;
-    type Writer<'a> = StoreWriter<'a, K, V>;
-
-    async fn open(config: Self::Config) -> Result<Self::Owned> {
-        let once = Arc::new(OnceLock::new());
-        let store = Store::open(config).await?;
-        once.set(store).unwrap();
-        Ok(Self {
-            once,
-            none: Store::None,
-        })
-    }
-
-    async fn close(&self) -> Result<()> {
-        match self.once.get() {
-            Some(store) => store.close().await,
-            None => self.none.close().await,
-        }
-    }
-
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
-        match self.once.get() {
-            Some(store) => store.writer(key, weight),
-            None => self.none.writer(key, weight),
-        }
-    }
-
-    fn exists(&self, key: &Self::Key) -> Result<bool> {
-        match self.once.get() {
-            Some(store) => store.exists(key),
-            None => self.none.exists(key),
-        }
-    }
-
-    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
-        match self.once.get() {
-            Some(store) => store.lookup(key).await,
-            None => self.none.lookup(key).await,
-        }
-    }
-
-    fn remove(&self, key: &Self::Key) -> Result<bool> {
-        match self.once.get() {
-            Some(store) => store.remove(key),
-            None => self.none.remove(key),
-        }
-    }
-
-    fn clear(&self) -> Result<()> {
-        match self.once.get() {
-            Some(store) => store.clear(),
-            None => self.none.clear(),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{path::PathBuf, time::Duration};
-
-    use foyer_intrusive::eviction::fifo::FifoConfig;
-
-    use crate::device::fs::FsDeviceConfig;
-
-    use super::*;
-
-    const KB: usize = 1024;
-    const MB: usize = 1024 * 1024;
-
-    #[tokio::test]
-    async fn test_lazy_store() {
-        let tempdir = tempfile::tempdir().unwrap();
-
-        let config = FifoFsStoreConfig {
-            name: "".to_string(),
-            eviction_config: FifoConfig,
-            device_config: FsDeviceConfig {
-                dir: PathBuf::from(tempdir.path()),
-                capacity: 16 * MB,
-                file_capacity: 4 * MB,
-                align: 4096,
-                io_size: 4096 * KB,
-            },
-            allocator_bits: 1,
-            admissions: vec![],
-            reinsertions: vec![],
-            buffer_pool_size: 8 * MB,
-            flushers: 1,
-            flush_rate_limit: 0,
-            reclaimers: 1,
-            reclaim_rate_limit: 0,
-            recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
-            clean_region_threshold: 1,
-        };
-
-        let (store, task) = LazyStore::lazy_with_task(config.into());
-
-        assert!(!store.insert(100, 100).await.unwrap());
-
-        tokio::spawn(task).await.unwrap().unwrap();
-
-        assert!(store.insert(100, 100).await.unwrap());
-        assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
-
-        store.close().await.unwrap();
-        drop(store);
-
-        let config = FifoFsStoreConfig {
-            name: "".to_string(),
-            eviction_config: FifoConfig,
-            device_config: FsDeviceConfig {
-                dir: PathBuf::from(tempdir.path()),
-                capacity: 16 * MB,
-                file_capacity: 4 * MB,
-                align: 4096,
-                io_size: 4096 * KB,
-            },
-            allocator_bits: 1,
-            admissions: vec![],
-            reinsertions: vec![],
-            buffer_pool_size: 8 * MB,
-            flushers: 1,
-            flush_rate_limit: 0,
-            reclaimers: 1,
-            reclaim_rate_limit: 0,
-            recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
-            clean_region_threshold: 1,
-        };
-
-        let (store, task) = LazyStore::lazy_with_task(config.into());
-
-        assert!(store.lookup(&100).await.unwrap().is_none());
-
-        tokio::spawn(task).await.unwrap().unwrap();
-
-        assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
-    }
-}
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 24ced44d..4c9a4b86 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -17,11 +17,11 @@ use std::{sync::Arc, time::Duration};
 use crate::{
     device::Device,
     error::Result,
+    generic::{GenericStore, RegionEntryIter},
     judge::Judges,
     metrics::Metrics,
     region_manager::{RegionEpItemAdapter, RegionManager},
-    store::{RegionEntryIter, Store},
-    Storage,
+    storage::Storage,
 };
 use bytes::BufMut;
 use foyer_common::{
@@ -42,7 +42,7 @@ where
 {
     threshold: usize,
 
-    store: Arc<Store<K, V, D, EP, EL>>,
+    store: Arc<GenericStore<K, V, D, EP, EL>>,
 
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
@@ -63,7 +63,7 @@ where
 {
     pub fn new(
         threshold: usize,
-        store: Arc<Store<K, V, D, EP, EL>>,
+        store: Arc<GenericStore<K, V, D, EP, EL>>,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         rate_limiter: Option<Arc<RateLimiter>>,
         metrics: Arc<Metrics>,
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
new file mode 100644
index 00000000..7da016cc
--- /dev/null
+++ b/foyer-storage/src/storage.rs
@@ -0,0 +1,286 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::fmt::Debug;
+
+use foyer_common::code::{Key, Value};
+
+use crate::error::Result;
+use futures::Future;
+
+pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
+
+pub trait StorageWriter: Send + Sync + Debug {
+    type Key: Key;
+    type Value: Value;
+
+    fn judge(&mut self) -> bool;
+
+    fn finish(self, value: Self::Value) -> impl Future<Output = Result<bool>> + Send;
+}
+
+pub trait Storage: Send + Sync + Debug + 'static {
+    type Key: Key;
+    type Value: Value;
+    type Config: Send + Debug;
+    type Owned: Send + Sync + Debug + 'static;
+    type Writer<'a>: StorageWriter<Key = Self::Key, Value = Self::Value>;
+
+    #[must_use]
+    fn open(config: Self::Config) -> impl Future<Output = Result<Self::Owned>> + Send;
+
+    #[must_use]
+    fn close(&self) -> impl Future<Output = Result<()>> + Send;
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_>;
+
+    fn exists(&self, key: &Self::Key) -> Result<bool>;
+
+    #[must_use]
+    fn lookup(&self, key: &Self::Key) -> impl Future<Output = Result<Option<Self::Value>>> + Send;
+
+    fn remove(&self, key: &Self::Key) -> Result<bool>;
+
+    fn clear(&self) -> Result<()>;
+}
+
+pub trait StorageExt: Storage {
+    #[must_use]
+    #[tracing::instrument(skip(self, value))]
+    fn insert(
+        &self,
+        key: Self::Key,
+        value: Self::Value,
+    ) -> impl Future<Output = Result<bool>> + Send {
+        let weight = key.serialized_len() + value.serialized_len();
+        self.writer(key, weight).finish(value)
+    }
+
+    #[must_use]
+    #[tracing::instrument(skip(self, value))]
+    fn insert_if_not_exists(
+        &self,
+        key: Self::Key,
+        value: Self::Value,
+    ) -> impl Future<Output = Result<bool>> + Send {
+        async move {
+            if self.exists(&key)? {
+                return Ok(false);
+            }
+            self.insert(key, value).await
+        }
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[must_use]
+    #[tracing::instrument(skip(self, f))]
+    fn insert_with<F>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f() {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            writer.finish(value).await
+        }
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called to fetch value, and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    fn insert_with_future<F, FU>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> FU + Send,
+        FU: FetchValueFuture<Self::Value>,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f().await {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            writer.finish(value).await
+        }
+    }
+
+    #[tracing::instrument(skip(self, f))]
+    fn insert_if_not_exists_with<F>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
+    {
+        async move {
+            if self.exists(&key)? {
+                return Ok(false);
+            }
+            self.insert_with(key, f, weight).await
+        }
+    }
+
+    #[tracing::instrument(skip(self, f))]
+    fn insert_if_not_exists_with_future<F, FU>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> FU + Send,
+        FU: FetchValueFuture<Self::Value>,
+    {
+        async move {
+            if self.exists(&key)? {
+                return Ok(false);
+            }
+            self.insert_with_future(key, f, weight).await
+        }
+    }
+}
+
+impl<S: Storage> StorageExt for S {}
+
+pub trait ForceStorageWriter: StorageWriter {
+    fn set_force(&mut self);
+}
+
+pub trait ForceStorageExt: Storage
+where
+    for<'w> Self::Writer<'w>: ForceStorageWriter,
+{
+    #[tracing::instrument(skip(self, value))]
+    fn insert_force(
+        &self,
+        key: Self::Key,
+        value: Self::Value,
+    ) -> impl Future<Output = Result<bool>> + Send {
+        let weight = key.serialized_len() + value.serialized_len();
+        let mut writer = self.writer(key, weight);
+        writer.set_force();
+        writer.finish(value)
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    fn insert_force_with<F>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> anyhow::Result<Self::Value> + Send,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            writer.set_force();
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f() {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            let inserted = writer.finish(value).await?;
+            Ok(inserted)
+        }
+    }
+
+    /// First judge if the entry will be admitted with `key` and `weight` by admission policies.
+    /// Then `f` will be called to fetch value, and entry will be inserted.
+    ///
+    /// # Safety
+    ///
+    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
+    #[tracing::instrument(skip(self, f))]
+    fn insert_force_with_future<F, FU>(
+        &self,
+        key: Self::Key,
+        f: F,
+        weight: usize,
+    ) -> impl Future<Output = Result<bool>> + Send
+    where
+        F: FnOnce() -> FU + Send,
+        FU: FetchValueFuture<Self::Value>,
+    {
+        async move {
+            let mut writer = self.writer(key, weight);
+            writer.set_force();
+            if !writer.judge() {
+                return Ok(false);
+            }
+            let value = match f().await {
+                Ok(value) => value,
+                Err(e) => {
+                    tracing::warn!("fetch value error: {:?}", e);
+                    return Ok(false);
+                }
+            };
+            let inserted = writer.finish(value).await?;
+            Ok(inserted)
+        }
+    }
+}
+
+impl<S> ForceStorageExt for S
+where
+    S: Storage,
+    for<'w> S::Writer<'w>: ForceStorageWriter,
+{
+}
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 14719615..6e90514c 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -12,1204 +12,289 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    marker::PhantomData,
-    sync::Arc,
-    time::{Duration, Instant},
-};
+use std::sync::Arc;
 
-use bitmaps::Bitmap;
-use bytes::{Buf, BufMut};
-use foyer_common::{bits, rate::RateLimiter};
-use foyer_intrusive::eviction::EvictionPolicy;
-use futures::{future::try_join_all, Future};
-use itertools::Itertools;
-use parking_lot::Mutex;
-use tokio::{sync::broadcast, task::JoinHandle};
-use twox_hash::XxHash64;
+use foyer_common::code::{Key, Value};
+use foyer_intrusive::eviction::{
+    fifo::{Fifo, FifoLink},
+    lfu::{Lfu, LfuLink},
+    lru::{Lru, LruLink},
+};
 
 use crate::{
-    admission::AdmissionPolicy,
-    device::Device,
+    device::fs::FsDevice,
     error::Result,
-    flusher::Flusher,
-    indices::{Index, Indices},
-    judge::Judges,
-    metrics::{Metrics, METRICS},
-    reclaimer::Reclaimer,
-    region::{Region, RegionHeader, RegionId, REGION_MAGIC},
-    region_manager::{RegionEpItemAdapter, RegionManager},
-    reinsertion::ReinsertionPolicy,
-    ForceStorageWriter, Storage, StorageWriter,
+    generic::{GenericStore, GenericStoreConfig, GenericStoreWriter},
+    region_manager::RegionEpItemAdapter,
+    storage::{ForceStorageWriter, Storage, StorageWriter},
 };
-use foyer_common::code::{Key, Value};
-use foyer_intrusive::core::adapter::Link;
-use std::hash::Hasher;
-
-const DEFAULT_BROADCAST_CAPACITY: usize = 4096;
-
-pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
-
-pub struct StoreConfig<K, V, D, EP>
-where
-    K: Key,
-    V: Value,
-    D: Device,
-    EP: EvictionPolicy,
-{
-    /// For distinguish different foyer metrics.
-    ///
-    /// Metrics of this foyer instance has label `foyer = {{ name }}`.
-    pub name: String,
-
-    /// Evictino policy configurations.
-    pub eviction_config: EP::Config,
-
-    /// Device configurations.
-    pub device_config: D::Config,
-
-    /// The count of allocators is `2 ^ allocator bits`.
-    ///
-    /// Note: The count of allocators should be greater than buffer count.
-    ///       (buffer count = buffer pool size / device region size)
-    pub allocator_bits: usize,
 
-    /// Admission policies.
-    pub admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
+pub type LruFsStore<K, V> =
+    GenericStore<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
 
-    /// Reinsertion policies.
-    pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
+pub type LruFsStoreConfig<K, V> =
+    GenericStoreConfig<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>>;
 
-    /// Buffer pool size, should be a multiplier of device region size.
-    pub buffer_pool_size: usize,
+pub type LruFsStoreWriter<'w, K, V> =
+    GenericStoreWriter<'w, K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
 
-    /// Count of flushers.
-    pub flushers: usize,
+pub type LfuFsStore<K, V> =
+    GenericStore<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
 
-    /// Flush rate limits.
-    pub flush_rate_limit: usize,
+pub type LfuFsStoreConfig<K, V> =
+    GenericStoreConfig<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>>;
 
-    /// Count of reclaimers.
-    pub reclaimers: usize,
+pub type LfuFsStoreWriter<'w, K, V> =
+    GenericStoreWriter<'w, K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
 
-    /// Flush rate limits.
-    pub reclaim_rate_limit: usize,
+pub type FifoFsStore<K, V> =
+    GenericStore<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
-    /// Allocation timout for skippable writers.
-    pub allocation_timeout: Duration,
+pub type FifoFsStoreConfig<K, V> =
+    GenericStoreConfig<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
-    /// Clean region count threshold to trigger reclamation.
-    ///
-    /// `clean_region_threshold` is recommended to be equal or larger than `reclaimers`.
-    pub clean_region_threshold: usize,
-
-    /// Concurrency of recovery.
-    pub recover_concurrency: usize,
-}
-
-impl<K, V, D, EP> Debug for StoreConfig<K, V, D, EP>
-where
-    K: Key,
-    V: Value,
-    D: Device,
-    EP: EvictionPolicy,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("StoreConfig")
-            .field("eviction_config", &self.eviction_config)
-            .field("device_config", &self.device_config)
-            .field("allocator_bits", &self.allocator_bits)
-            .field("admissions", &self.admissions)
-            .field("reinsertions", &self.reinsertions)
-            .field("buffer_pool_size", &self.buffer_pool_size)
-            .field("flushers", &self.flushers)
-            .field("flush_rate_limit", &self.flush_rate_limit)
-            .field("reclaimers", &self.reclaimers)
-            .field("reclaim_rate_limit", &self.reclaim_rate_limit)
-            .field("allocation_timeout", &self.allocation_timeout)
-            .field("clean_region_threshold", &self.clean_region_threshold)
-            .field("recover_concurrency", &self.recover_concurrency)
-            .finish()
-    }
-}
+pub type FifoFsStoreWriter<'w, K, V> =
+    GenericStoreWriter<'w, K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
 #[derive(Debug)]
-pub struct Store<K, V, D, EP, EL>
+pub enum StoreConfig<K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
-    indices: Arc<Indices<K>>,
-
-    region_manager: Arc<RegionManager<D, EP, EL>>,
-
-    device: D,
-
-    admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
-    reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
-
-    flusher_handles: Mutex<Vec<JoinHandle<()>>>,
-    flushers_stop_tx: broadcast::Sender<()>,
-
-    reclaimer_handles: Mutex<Vec<JoinHandle<()>>>,
-    reclaimers_stop_tx: broadcast::Sender<()>,
-
-    metrics: Arc<Metrics>,
-
-    _marker: PhantomData<V>,
+    LruFsStoreConfig { config: LruFsStoreConfig<K, V> },
+    LfuFsStoreConfig { config: LfuFsStoreConfig<K, V> },
+    FifoFsStoreConfig { config: FifoFsStoreConfig<K, V> },
+    None,
 }
 
-impl<K, V, D, EP, EL> Store<K, V, D, EP, EL>
+impl<K, V> From<LruFsStoreConfig<K, V>> for StoreConfig<K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
-    async fn open(config: StoreConfig<K, V, D, EP>) -> Result<Arc<Self>> {
-        tracing::info!("open store with config:\n{:#?}", config);
-
-        let metrics = Arc::new(METRICS.foyer(&config.name));
-
-        let device = D::open(config.device_config).await?;
-
-        let buffer_count = config.buffer_pool_size / device.region_size();
-
-        if buffer_count < (1 << config.allocator_bits) {
-            return Err(anyhow::anyhow!(
-                "The count of allocators shoule be greater than buffer count."
-            )
-            .into());
-        }
-
-        let region_manager = Arc::new(RegionManager::new(
-            config.allocator_bits,
-            buffer_count,
-            device.regions(),
-            config.eviction_config,
-            device.clone(),
-            config.allocation_timeout,
-            metrics.clone(),
-        ));
-
-        let indices = Arc::new(Indices::new(device.regions()));
-
-        let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
-        let (reclaimers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
-
-        let flusher_stop_rxs = (0..config.flushers)
-            .map(|_| flushers_stop_tx.subscribe())
-            .collect_vec();
-        let reclaimer_stop_rxs = (0..config.reclaimers)
-            .map(|_| reclaimers_stop_tx.subscribe())
-            .collect_vec();
-
-        let store = Arc::new(Self {
-            indices: indices.clone(),
-            region_manager: region_manager.clone(),
-            device: device.clone(),
-            admissions: config.admissions,
-            reinsertions: config.reinsertions,
-            flusher_handles: Mutex::new(vec![]),
-            reclaimer_handles: Mutex::new(vec![]),
-            flushers_stop_tx,
-            reclaimers_stop_tx,
-            metrics: metrics.clone(),
-            _marker: PhantomData,
-        });
-
-        for admission in store.admissions.iter() {
-            admission.init(&store.indices);
-        }
-        for reinsertion in store.reinsertions.iter() {
-            reinsertion.init(&store.indices);
-        }
-
-        let flush_rate_limiter = match config.flush_rate_limit {
-            0 => None,
-            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
-        };
-        let reclaim_rate_limiter = match config.reclaim_rate_limit {
-            0 => None,
-            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
-        };
-
-        let flushers = flusher_stop_rxs
-            .into_iter()
-            .map(|stop_rx| {
-                Flusher::new(
-                    region_manager.clone(),
-                    flush_rate_limiter.clone(),
-                    metrics.clone(),
-                    stop_rx,
-                )
-            })
-            .collect_vec();
-        let reclaimers = reclaimer_stop_rxs
-            .into_iter()
-            .map(|stop_rx| {
-                Reclaimer::new(
-                    config.clean_region_threshold,
-                    store.clone(),
-                    region_manager.clone(),
-                    reclaim_rate_limiter.clone(),
-                    metrics.clone(),
-                    stop_rx,
-                )
-            })
-            .collect_vec();
-
-        store.recover(config.recover_concurrency).await?;
-
-        let flusher_handles = flushers
-            .into_iter()
-            .map(|flusher| tokio::spawn(async move { flusher.run().await.unwrap() }))
-            .collect_vec();
-
-        let reclaimer_handles = reclaimers
-            .into_iter()
-            .map(|reclaimer| tokio::spawn(async move { reclaimer.run().await.unwrap() }))
-            .collect_vec();
-
-        *store.flusher_handles.lock() = flusher_handles;
-        *store.reclaimer_handles.lock() = reclaimer_handles;
-
-        Ok(store)
-    }
-
-    async fn close(&self) -> Result<()> {
-        // seal current dirty buffer and trigger flushing
-        self.seal().await;
-
-        // stop and wait for reclaimers
-        let handles = self.reclaimer_handles.lock().drain(..).collect_vec();
-        if !handles.is_empty() {
-            self.reclaimers_stop_tx.send(()).unwrap();
-        }
-        for handle in handles {
-            handle.await.unwrap();
-        }
-
-        // stop and wait for flushers
-        let handles = self.flusher_handles.lock().drain(..).collect_vec();
-        if !handles.is_empty() {
-            self.flushers_stop_tx.send(()).unwrap();
-        }
-        for handle in handles {
-            handle.await.unwrap();
-        }
-
-        Ok(())
-    }
-
-    /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
-    #[tracing::instrument(skip(self))]
-    fn writer(&self, key: K, weight: usize) -> StoreWriter<'_, K, V, D, EP, EL> {
-        StoreWriter::new(self, key, weight)
-    }
-
-    #[tracing::instrument(skip(self))]
-    fn exists(&self, key: &K) -> Result<bool> {
-        Ok(self.indices.lookup(key).is_some())
-    }
-
-    #[tracing::instrument(skip(self))]
-    async fn lookup(&self, key: &K) -> Result<Option<V>> {
-        let now = Instant::now();
-
-        let index = match self.indices.lookup(key) {
-            Some(index) => index,
-            None => {
-                self.metrics
-                    .op_duration_lookup_miss
-                    .observe(now.elapsed().as_secs_f64());
-                return Ok(None);
-            }
-        };
-
-        self.region_manager.record_access(&index.region);
-        let region = self.region_manager.region(&index.region);
-        let start = index.offset as usize;
-        let end = start + index.len as usize;
-
-        // TODO(MrCroxx): read value only
-        let slice = match region.load(start..end, index.version).await? {
-            Some(slice) => slice,
-            None => {
-                // Remove index if the storage layer fails to lookup it (because of region version mismatch).
-                self.indices.remove(key);
-                self.metrics
-                    .op_duration_lookup_miss
-                    .observe(now.elapsed().as_secs_f64());
-                return Ok(None);
-            }
-        };
-        self.metrics.op_bytes_lookup.inc_by(slice.len() as u64);
-
-        let res = match read_entry::<K, V>(slice.as_ref()) {
-            Some((_key, value)) => Ok(Some(value)),
-            None => {
-                // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
-                self.indices.remove(key);
-                Ok(None)
-            }
-        };
-        drop(slice);
-
-        self.metrics
-            .op_duration_lookup_hit
-            .observe(now.elapsed().as_secs_f64());
-
-        res
-    }
-
-    #[tracing::instrument(skip(self))]
-    fn remove(&self, key: &K) -> Result<bool> {
-        let _timer = self.metrics.op_duration_remove.start_timer();
-
-        let res = self.indices.remove(key).is_some();
-
-        Ok(res)
-    }
-
-    #[tracing::instrument(skip(self))]
-    fn clear(&self) -> Result<()> {
-        self.indices.clear();
-
-        // TODO(MrCroxx): set all regions as clean?
-
-        Ok(())
-    }
-
-    pub(crate) fn indices(&self) -> &Arc<Indices<K>> {
-        &self.indices
-    }
-
-    pub(crate) fn reinsertions(&self) -> &Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>> {
-        &self.reinsertions
-    }
-
-    fn serialized_len(&self, key: &K, value: &V) -> usize {
-        let unaligned =
-            EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len();
-        bits::align_up(self.device.align(), unaligned)
-    }
-
-    async fn seal(&self) {
-        self.region_manager.seal().await;
-    }
-
-    #[tracing::instrument(skip(self))]
-    async fn recover(&self, concurrency: usize) -> Result<()> {
-        tracing::info!("start store recovery");
-
-        let (tx, rx) = async_channel::bounded(concurrency);
-
-        let mut handles = vec![];
-        for region_id in 0..self.device.regions() as RegionId {
-            let itx = tx.clone();
-            let irx = rx.clone();
-            let region_manager = self.region_manager.clone();
-            let indices = self.indices.clone();
-            let handle = tokio::spawn(async move {
-                itx.send(()).await.unwrap();
-                let res = Self::recover_region(region_id, region_manager, indices).await;
-                irx.recv().await.unwrap();
-                res
-            });
-            handles.push(handle);
-        }
-
-        let mut recovered = 0;
-
-        let results = try_join_all(handles).await.map_err(anyhow::Error::from)?;
-
-        for (region_id, result) in results.into_iter().enumerate() {
-            if result? {
-                tracing::debug!("region {} is recovered", region_id);
-                recovered += 1
-            }
-        }
-
-        tracing::info!("finish store recovery, {} region recovered", recovered);
-        self.metrics
-            .total_bytes
-            .set((recovered * self.device.region_size()) as u64);
-
-        // Force trigger reclamation.
-        if recovered == self.device.regions() {
-            self.region_manager.clean_regions().flash();
-        }
-
-        Ok(())
-    }
-
-    /// Return `true` if region is valid, otherwise `false`
-    async fn recover_region(
-        region_id: RegionId,
-        region_manager: Arc<RegionManager<D, EP, EL>>,
-        indices: Arc<Indices<K>>,
-    ) -> Result<bool> {
-        let region = region_manager.region(&region_id).clone();
-        let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
-            while let Some(index) = iter.next().await? {
-                indices.insert(index);
-            }
-            region_manager.eviction_push(region_id);
-            true
-        } else {
-            region_manager.clean_regions().release(region_id);
-            false
-        };
-        Ok(res)
-    }
-
-    fn judge_inner(&self, writer: &mut StoreWriter<'_, K, V, D, EP, EL>) {
-        for (index, admission) in self.admissions.iter().enumerate() {
-            let judge = admission.judge(&writer.key, writer.weight, &self.metrics);
-            writer.judges.set(index, judge);
-        }
-        writer.is_judged = true;
-    }
-
-    async fn apply_writer(
-        &self,
-        mut writer: StoreWriter<'_, K, V, D, EP, EL>,
-        value: V,
-    ) -> Result<bool> {
-        debug_assert!(!writer.is_inserted);
-
-        if !writer.judge() {
-            return Ok(false);
-        }
-
-        let now = Instant::now();
-
-        writer.is_inserted = true;
-        let key = &writer.key;
-
-        for (i, admission) in self.admissions.iter().enumerate() {
-            let judge = writer.judges.get(i);
-            admission.on_insert(key, writer.weight, &self.metrics, judge);
-        }
-
-        let serialized_len = self.serialized_len(key, &value);
-
-        if key.serialized_len() + value.serialized_len() != writer.weight {
-            tracing::error!(
-                "weight != key.serialized_len() + value.serialized_len(), weight: {}, key size: {}, value size: {}, key: {:?}",
-                writer.weight, key.serialized_len(), value.serialized_len(), key
-            );
-        }
-
-        self.metrics.op_bytes_insert.inc_by(serialized_len as u64);
-
-        let mut slice = match self
-            .region_manager
-            .allocate(serialized_len, !writer.is_skippable)
-            .await
-        {
-            Some(slice) => slice,
-            // Only reachable when writer is skippable.
-            None => return Ok(false),
-        };
-
-        write_entry(slice.as_mut(), key, &value);
-
-        let index = Index {
-            region: slice.region_id(),
-            version: slice.version(),
-            offset: slice.offset() as u32,
-            len: slice.len() as u32,
-            key_len: key.serialized_len() as u32,
-            value_len: value.serialized_len() as u32,
-
-            key: key.clone(),
-        };
-        drop(slice);
-
-        self.indices.insert(index);
-
-        let duration = now.elapsed() + writer.duration;
-        self.metrics
-            .op_duration_insert_inserted
-            .observe(duration.as_secs_f64());
-
-        Ok(true)
+    fn from(config: LruFsStoreConfig<K, V>) -> Self {
+        StoreConfig::LruFsStoreConfig { config }
     }
 }
 
-pub struct StoreWriter<'a, K, V, D, EP, EL>
+impl<K, V> From<LfuFsStoreConfig<K, V>> for StoreConfig<K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
-    store: &'a Store<K, V, D, EP, EL>,
-    key: K,
-    weight: usize,
-
-    judges: Judges,
-    is_judged: bool,
-
-    /// judge duration
-    duration: Duration,
-
-    is_inserted: bool,
-    is_skippable: bool,
+    fn from(config: LfuFsStoreConfig<K, V>) -> Self {
+        StoreConfig::LfuFsStoreConfig { config }
+    }
 }
 
-impl<'a, K, V, D, EP, EL> StoreWriter<'a, K, V, D, EP, EL>
+impl<K, V> From<FifoFsStoreConfig<K, V>> for StoreConfig<K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
-    fn new(store: &'a Store<K, V, D, EP, EL>, key: K, weight: usize) -> Self {
-        Self {
-            store,
-            key,
-            weight,
-            judges: Judges::new(store.admissions.len()),
-            is_judged: false,
-            duration: Duration::from_nanos(0),
-            is_inserted: false,
-            is_skippable: false,
-        }
-    }
-
-    /// Judge if the entry can be admitted by configured admission policies.
-    pub fn judge(&mut self) -> bool {
-        if !self.is_judged {
-            let now = Instant::now();
-            self.store.judge_inner(self);
-            self.duration = now.elapsed();
-        }
-        self.judges.judge()
-    }
-
-    pub async fn finish(self, value: V) -> Result<bool> {
-        self.store.apply_writer(self, value).await
-    }
-
-    pub fn set_force(&mut self) {
-        self.judges.set_mask(Bitmap::new());
-    }
-
-    pub fn set_judge_mask(&mut self, mask: Bitmap<64>) {
-        self.judges.set_mask(mask);
-    }
-
-    pub fn set_skippable(&mut self) {
-        self.is_skippable = true
+    fn from(config: FifoFsStoreConfig<K, V>) -> Self {
+        StoreConfig::FifoFsStoreConfig { config }
     }
 }
 
-impl<'a, K, V, D, EP, EL> Debug for StoreWriter<'a, K, V, D, EP, EL>
+#[derive(Debug)]
+pub enum StoreWriter<'a, K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("StoreWriter")
-            .field("key", &self.key)
-            .field("weight", &self.weight)
-            .field("judges", &self.judges)
-            .field("is_judged", &self.is_judged)
-            .field("duration", &self.duration)
-            .field("inserted", &self.is_inserted)
-            .finish()
-    }
+    LruFsStorWriter { writer: LruFsStoreWriter<'a, K, V> },
+    LfuFsStorWriter { writer: LfuFsStoreWriter<'a, K, V> },
+    FifoFsStoreWriter { writer: FifoFsStoreWriter<'a, K, V> },
+    None,
 }
 
-impl<'a, K, V, D, EP, EL> Drop for StoreWriter<'a, K, V, D, EP, EL>
+impl<'a, K, V> From<LruFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
-    fn drop(&mut self) {
-        if !self.is_inserted {
-            self.store
-                .metrics
-                .op_duration_insert_dropped
-                .observe(self.duration.as_secs_f64());
-            let mut filtered = false;
-            if self.is_judged {
-                for (i, admission) in self.store.admissions.iter().enumerate() {
-                    let judge = self.judges.get(i);
-                    admission.on_drop(&self.key, self.weight, &self.store.metrics, judge);
-                }
-                filtered = !self.judge();
-            }
-            if filtered {
-                self.store
-                    .metrics
-                    .op_duration_insert_filtered
-                    .observe(self.duration.as_secs_f64());
-            } else {
-                self.store
-                    .metrics
-                    .op_duration_insert_dropped
-                    .observe(self.duration.as_secs_f64());
-            }
-        }
+    fn from(writer: LruFsStoreWriter<'a, K, V>) -> Self {
+        StoreWriter::LruFsStorWriter { writer }
     }
 }
 
-const ENTRY_MAGIC: u32 = 0x97_00_00_00;
-const ENTRY_MAGIC_MASK: u32 = 0xFF_00_00_00;
-
-#[derive(Debug)]
-struct EntryHeader {
-    key_len: u32,
-    value_len: u32,
-    checksum: u64,
-}
-
-impl EntryHeader {
-    fn serialized_len() -> usize {
-        4 + 4 + 8
-    }
-
-    fn write(&self, mut buf: &mut [u8]) {
-        buf.put_u32(self.key_len | ENTRY_MAGIC);
-        buf.put_u32(self.value_len);
-        buf.put_u64(self.checksum);
-    }
-
-    fn read(mut buf: &[u8]) -> Option<Self> {
-        let head = buf.get_u32();
-        let magic = head & ENTRY_MAGIC_MASK;
-
-        if magic != ENTRY_MAGIC {
-            return None;
-        }
-
-        let key_len = head ^ ENTRY_MAGIC;
-        let value_len = buf.get_u32();
-        let checksum = buf.get_u64();
-
-        Some(Self {
-            key_len,
-            value_len,
-            checksum,
-        })
-    }
-}
-
-/// | header | value | key | <padding> |
-///
-/// # Safety
-///
-/// `buf.len()` must excatly fit entry size
-fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V)
+impl<'a, K, V> From<LfuFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
 where
     K: Key,
     V: Value,
 {
-    let mut offset = EntryHeader::serialized_len();
-    value.write(&mut buf[offset..offset + value.serialized_len()]);
-    offset += value.serialized_len();
-    key.write(&mut buf[offset..offset + key.serialized_len()]);
-    offset += key.serialized_len();
-    let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
-
-    let header = EntryHeader {
-        key_len: key.serialized_len() as u32,
-        value_len: value.serialized_len() as u32,
-        checksum,
-    };
-    header.write(&mut buf[..EntryHeader::serialized_len()]);
+    fn from(writer: LfuFsStoreWriter<'a, K, V>) -> Self {
+        StoreWriter::LfuFsStorWriter { writer }
+    }
 }
 
-/// | header | value | key | <padding> |
-///
-/// # Safety
-///
-/// `buf.len()` must excatly fit entry size
-fn read_entry<K, V>(buf: &[u8]) -> Option<(K, V)>
+impl<'a, K, V> From<FifoFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
 where
     K: Key,
     V: Value,
 {
-    let header = EntryHeader::read(buf)?;
-
-    let mut offset = EntryHeader::serialized_len();
-    let value = V::read(&buf[offset..offset + header.value_len as usize]);
-    offset += header.value_len as usize;
-    let key = K::read(&buf[offset..offset + header.key_len as usize]);
-    offset += header.key_len as usize;
-
-    let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
-    if checksum != header.checksum {
-        tracing::warn!(
-            "checksum mismatch, checksum: {}, expected: {}",
-            checksum,
-            header.checksum,
-        );
-        return None;
+    fn from(writer: FifoFsStoreWriter<'a, K, V>) -> Self {
+        StoreWriter::FifoFsStoreWriter { writer }
     }
-
-    Some((key, value))
 }
 
-fn checksum(buf: &[u8]) -> u64 {
-    let mut hasher = XxHash64::with_seed(0);
-    hasher.write(buf);
-    hasher.finish()
-}
-
-pub struct RegionEntryIter<K, V, D>
+#[derive(Debug)]
+pub enum Store<K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
 {
-    region: Region<D>,
-
-    cursor: usize,
-
-    _marker: PhantomData<(K, V)>,
+    LruFsStore { store: Arc<LruFsStore<K, V>> },
+    LfuFsStore { store: Arc<LfuFsStore<K, V>> },
+    FifoFsStore { store: Arc<FifoFsStore<K, V>> },
+    None,
 }
 
-impl<K, V, D> RegionEntryIter<K, V, D>
+impl<K, V> Clone for Store<K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
 {
-    pub async fn open(region: Region<D>) -> Result<Option<Self>> {
-        let align = region.device().align();
-
-        let slice = match region.load(..align, 0).await? {
-            Some(slice) => slice,
-            None => return Ok(None),
-        };
-
-        let header = RegionHeader::read(slice.as_ref());
-        drop(slice);
-
-        if header.magic != REGION_MAGIC {
-            return Ok(None);
-        }
-
-        Ok(Some(Self {
-            region,
-            cursor: align,
-            _marker: PhantomData,
-        }))
-    }
-
-    pub async fn next(&mut self) -> Result<Option<Index<K>>> {
-        let region_size = self.region.device().region_size();
-        let align = self.region.device().align();
-
-        if self.cursor + align >= region_size {
-            return Ok(None);
-        }
-
-        let Some(slice) = self
-            .region
-            .load(self.cursor..self.cursor + align, 0)
-            .await?
-        else {
-            return Ok(None);
-        };
-
-        let Some(header) = EntryHeader::read(slice.as_ref()) else {
-            return Ok(None);
-        };
-
-        let entry_len = bits::align_up(
-            align,
-            (header.value_len + header.key_len) as usize + EntryHeader::serialized_len(),
-        );
-
-        let abs_start = self.cursor + EntryHeader::serialized_len() + header.value_len as usize;
-        let abs_end = self.cursor
-            + EntryHeader::serialized_len()
-            + (header.key_len + header.value_len) as usize;
-
-        if abs_start >= abs_end || abs_end > region_size {
-            // Double check wrong entry.
-            return Ok(None);
+    fn clone(&self) -> Self {
+        match self {
+            Self::LruFsStore { store } => Self::LruFsStore {
+                store: Arc::clone(store),
+            },
+            Self::LfuFsStore { store } => Self::LfuFsStore {
+                store: Arc::clone(store),
+            },
+            Self::FifoFsStore { store } => Self::FifoFsStore {
+                store: Arc::clone(store),
+            },
+            Self::None => Self::None,
         }
-
-        let align_start = bits::align_down(align, abs_start);
-        let align_end = bits::align_up(align, abs_end);
-
-        let key = if align_start == self.cursor - align && align_end == self.cursor {
-            // header and key are in the same block, read directly from slice
-            let rel_start = EntryHeader::serialized_len() + header.value_len as usize;
-            let rel_end = rel_start + header.key_len as usize;
-            let key = K::read(&slice.as_ref()[rel_start..rel_end]);
-            drop(slice);
-            key
-        } else {
-            drop(slice);
-            let Some(s) = self.region.load(align_start..align_end, 0).await? else {
-                return Ok(None);
-            };
-            let rel_start = abs_start - align_start;
-            let rel_end = abs_end - align_start;
-
-            let key = K::read(&s.as_ref()[rel_start..rel_end]);
-            drop(s);
-            key
-        };
-
-        let index = Index {
-            key,
-            region: self.region.id(),
-            version: 0,
-            offset: self.cursor as u32,
-            len: entry_len as u32,
-            key_len: header.key_len,
-            value_len: header.value_len,
-        };
-
-        self.cursor += entry_len;
-
-        Ok(Some(index))
-    }
-
-    pub async fn next_kv(&mut self) -> Result<Option<(K, V)>> {
-        let index = match self.next().await {
-            Ok(Some(index)) => index,
-            Ok(None) => return Ok(None),
-            Err(e) => return Err(e),
-        };
-
-        // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
-        let start = index.offset as usize;
-        let end = start + index.len as usize;
-        let Some(slice) = self.region.load(start..end, 0).await? else {
-            return Ok(None);
-        };
-        let kv = read_entry::<K, V>(slice.as_ref());
-        drop(slice);
-
-        Ok(kv)
     }
 }
 
-impl<'a, K, V, D, EP, EL> StorageWriter for StoreWriter<'a, K, V, D, EP, EL>
+impl<'a, K, V> StorageWriter for StoreWriter<'a, K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
     type Key = K;
     type Value = V;
 
     fn judge(&mut self) -> bool {
-        self.judge()
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.judge(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.judge(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.judge(),
+            StoreWriter::None => false,
+        }
     }
 
     async fn finish(self, value: Self::Value) -> Result<bool> {
-        self.finish(value).await
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.finish(value).await,
+            StoreWriter::LfuFsStorWriter { writer } => writer.finish(value).await,
+            StoreWriter::FifoFsStoreWriter { writer } => writer.finish(value).await,
+            StoreWriter::None => Ok(false),
+        }
     }
 }
 
-impl<'a, K, V, D, EP, EL> ForceStorageWriter for StoreWriter<'a, K, V, D, EP, EL>
+impl<'a, K, V> ForceStorageWriter for StoreWriter<'a, K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
     fn set_force(&mut self) {
-        self.set_force()
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.set_force(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.set_force(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.set_force(),
+            StoreWriter::None => {}
+        }
     }
 }
 
-impl<K, V, D, EP, EL> Storage for Store<K, V, D, EP, EL>
+impl<K, V> Storage for Store<K, V>
 where
     K: Key,
     V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
 {
     type Key = K;
     type Value = V;
-    type Config = StoreConfig<K, V, D, EP>;
-    type Owned = Arc<Self>;
-    type Writer<'a> = StoreWriter<'a, K, V, D, EP, EL>;
+    type Config = StoreConfig<K, V>;
+    type Owned = Self;
+    type Writer<'a> = StoreWriter<'a, K, V>;
 
     async fn open(config: Self::Config) -> Result<Self::Owned> {
-        Self::open(config).await
+        match config {
+            StoreConfig::LruFsStoreConfig { config } => {
+                let store = LruFsStore::open(config).await?;
+                Ok(Self::LruFsStore { store })
+            }
+            StoreConfig::LfuFsStoreConfig { config } => {
+                let store = LfuFsStore::open(config).await?;
+                Ok(Self::LfuFsStore { store })
+            }
+            StoreConfig::FifoFsStoreConfig { config } => {
+                let store = FifoFsStore::open(config).await?;
+                Ok(Self::FifoFsStore { store })
+            }
+            StoreConfig::None => Ok(Self::None),
+        }
     }
 
     async fn close(&self) -> Result<()> {
-        self.close().await
+        match self {
+            Store::LruFsStore { store } => store.close().await,
+            Store::LfuFsStore { store } => store.close().await,
+            Store::FifoFsStore { store } => store.close().await,
+            Store::None => Ok(()),
+        }
     }
 
     fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
-        self.writer(key, weight)
-    }
-
-    fn exists(&self, key: &Self::Key) -> Result<bool> {
-        self.exists(key)
-    }
-
-    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
-        self.lookup(key).await
-    }
-
-    fn remove(&self, key: &Self::Key) -> Result<bool> {
-        self.remove(key)
-    }
-
-    fn clear(&self) -> Result<()> {
-        self.clear()
-    }
-}
-
-#[cfg(test)]
-pub mod tests {
-    use std::{collections::HashSet, path::PathBuf};
-
-    use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
-
-    use crate::{
-        device::fs::{FsDevice, FsDeviceConfig},
-        StorageExt,
-    };
-
-    use super::*;
-
-    type TestStore = Store<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
-
-    type TestStoreConfig = StoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
-
-    #[derive(Debug, Clone)]
-    enum Record<K: Key> {
-        Admit(K),
-        Evict(K),
-    }
-
-    #[derive(Debug)]
-    struct JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        records: Mutex<Vec<Record<K>>>,
-        _marker: PhantomData<V>,
-    }
-
-    impl<K, V> JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        fn dump(&self) -> Vec<Record<K>> {
-            self.records.lock().clone()
-        }
-
-        fn remains(&self) -> HashSet<K> {
-            let records = self.dump();
-            let mut res = HashSet::default();
-            for record in records {
-                match record {
-                    Record::Admit(key) => {
-                        res.insert(key);
-                    }
-                    Record::Evict(key) => {
-                        res.remove(&key);
-                    }
-                }
-            }
-            res
+        match self {
+            Store::LruFsStore { store } => store.writer(key, weight).into(),
+            Store::LfuFsStore { store } => store.writer(key, weight).into(),
+            Store::FifoFsStore { store } => store.writer(key, weight).into(),
+            Store::None => StoreWriter::None,
         }
     }
 
-    impl<K, V> Default for JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        fn default() -> Self {
-            Self {
-                records: Mutex::new(Vec::default()),
-                _marker: PhantomData,
-            }
+    fn exists(&self, key: &Self::Key) -> Result<bool> {
+        match self {
+            Store::LruFsStore { store } => store.exists(key),
+            Store::LfuFsStore { store } => store.exists(key),
+            Store::FifoFsStore { store } => store.exists(key),
+            Store::None => Ok(false),
         }
     }
 
-    impl<K, V> AdmissionPolicy for JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        type Key = K;
-
-        type Value = V;
-
-        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
-            self.records.lock().push(Record::Admit(key.clone()));
-            true
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        match self {
+            Store::LruFsStore { store } => store.lookup(key).await,
+            Store::LfuFsStore { store } => store.lookup(key).await,
+            Store::FifoFsStore { store } => store.lookup(key).await,
+            Store::None => Ok(None),
         }
-
-        fn on_insert(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
-
-        fn on_drop(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
     }
 
-    impl<K, V> ReinsertionPolicy for JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        type Key = K;
-
-        type Value = V;
-
-        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
-            self.records.lock().push(Record::Evict(key.clone()));
-            false
-        }
-
-        fn on_insert(
-            &self,
-            _key: &Self::Key,
-            _weight: usize,
-            _metrics: &Arc<crate::metrics::Metrics>,
-            _judge: bool,
-        ) {
-        }
-
-        fn on_drop(
-            &self,
-            _key: &Self::Key,
-            _weight: usize,
-            _metrics: &Arc<crate::metrics::Metrics>,
-            _judge: bool,
-        ) {
+    fn remove(&self, key: &Self::Key) -> Result<bool> {
+        match self {
+            Store::LruFsStore { store } => store.remove(key),
+            Store::LfuFsStore { store } => store.remove(key),
+            Store::FifoFsStore { store } => store.remove(key),
+            Store::None => Ok(false),
         }
     }
 
-    #[tokio::test]
-    #[expect(clippy::identity_op)]
-    async fn test_recovery() {
-        const KB: usize = 1024;
-        const MB: usize = 1024 * 1024;
-
-        let tempdir = tempfile::tempdir().unwrap();
-
-        let recorder = Arc::new(JudgeRecorder::default());
-        let admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> =
-            vec![recorder.clone()];
-        let reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> =
-            vec![recorder.clone()];
-
-        let config = TestStoreConfig {
-            name: "".to_string(),
-            eviction_config: FifoConfig,
-            device_config: FsDeviceConfig {
-                dir: PathBuf::from(tempdir.path()),
-                capacity: 16 * MB,
-                file_capacity: 4 * MB,
-                align: 4096,
-                io_size: 4096 * KB,
-            },
-            allocator_bits: 1,
-            admissions,
-            reinsertions,
-            buffer_pool_size: 8 * MB,
-            flushers: 1,
-            flush_rate_limit: 0,
-            reclaimers: 1,
-            reclaim_rate_limit: 0,
-            recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
-            clean_region_threshold: 1,
-        };
-
-        let store = TestStore::open(config).await.unwrap();
-
-        // files:
-        // [0, 1, 2]
-        // [3, 4, 5]
-        // [6, 7, 8]
-        // [9, 10, 11]
-        for i in 0..20 {
-            store.insert(i, vec![i as u8; 1 * MB]).await.unwrap();
-        }
-
-        store.close().await.unwrap();
-
-        let remains = recorder.remains();
-
-        for i in 0..20 {
-            if remains.contains(&i) {
-                assert_eq!(
-                    store.lookup(&i).await.unwrap().unwrap(),
-                    vec![i as u8; 1 * MB],
-                );
-            } else {
-                assert!(store.lookup(&i).await.unwrap().is_none());
-            }
-        }
-
-        drop(store);
-
-        let config = TestStoreConfig {
-            name: "".to_string(),
-            eviction_config: FifoConfig,
-            device_config: FsDeviceConfig {
-                dir: PathBuf::from(tempdir.path()),
-                capacity: 16 * MB,
-                file_capacity: 4 * MB,
-                align: 4096,
-                io_size: 4096 * KB,
-            },
-            allocator_bits: 1,
-            admissions: vec![],
-            reinsertions: vec![],
-            buffer_pool_size: 8 * MB,
-            flushers: 1,
-            flush_rate_limit: 0,
-            reclaimers: 0,
-            reclaim_rate_limit: 0,
-            recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
-            clean_region_threshold: 1,
-        };
-        let store = TestStore::open(config).await.unwrap();
-
-        for i in 0..12 {
-            if remains.contains(&i) {
-                assert_eq!(
-                    store.lookup(&i).await.unwrap().unwrap(),
-                    vec![i as u8; 1 * MB],
-                );
-            } else {
-                assert!(store.lookup(&i).await.unwrap().is_none());
-            }
+    fn clear(&self) -> Result<()> {
+        match self {
+            Store::LruFsStore { store } => store.clear(),
+            Store::LfuFsStore { store } => store.clear(),
+            Store::FifoFsStore { store } => store.clear(),
+            Store::None => Ok(()),
         }
-
-        store.close().await.unwrap();
-
-        drop(store);
     }
 }

From 974f7e53e1c79e8b8dd5ff39144b6ec5b26737d0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 7 Oct 2023 10:02:20 -0500
Subject: [PATCH 120/261] chore: remove unused (#153)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/generic.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index a9190a16..ec46a0a4 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -23,7 +23,7 @@ use bitmaps::Bitmap;
 use bytes::{Buf, BufMut};
 use foyer_common::{bits, rate::RateLimiter};
 use foyer_intrusive::eviction::EvictionPolicy;
-use futures::{future::try_join_all, Future};
+use futures::future::try_join_all;
 use itertools::Itertools;
 use parking_lot::Mutex;
 use tokio::{sync::broadcast, task::JoinHandle};
@@ -49,8 +49,6 @@ use std::hash::Hasher;
 
 const DEFAULT_BROADCAST_CAPACITY: usize = 4096;
 
-pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
-
 pub struct GenericStoreConfig<K, V, D, EP>
 where
     K: Key,

From 7ca1a247fc86c8a49cde57f5bc8c345b9303b6a2 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 7 Oct 2023 10:25:42 -0500
Subject: [PATCH 121/261] chore: fix lazy store clone (#154)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/lazy.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 7da1803f..3dc2a391 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -22,7 +22,7 @@ use crate::{
 use foyer_common::code::{Key, Value};
 use futures::Future;
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct LazyStore<K, V>
 where
     K: Key,
@@ -32,6 +32,19 @@ where
     none: Store<K, V>,
 }
 
+impl<K, V> Clone for LazyStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn clone(&self) -> Self {
+        Self {
+            once: Arc::clone(&self.once),
+            none: Store::None,
+        }
+    }
+}
+
 impl<K, V> LazyStore<K, V>
 where
     K: Key,

From ae56c1aa8eb9cdb9a3db51641a8b0f004e6d9296 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 7 Oct 2023 11:08:01 -0500
Subject: [PATCH 122/261] chore: add codecov config (#155)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 codecov.yml | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 codecov.yml

diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 00000000..0d75e0fd
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,6 @@
+# codecov config
+# Reference: https://docs.codecov.com/docs/codecovyml-reference
+# Tips. You may run following command to validate before committing any changes
+#   curl --data-binary @codecov.yml https://codecov.io/validate
+ignore:
+  - "foyer-storage-bench"

From 882d4973d96859133b9fe3ee0d5db50a64941819 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 8 Oct 2023 03:55:30 -0500
Subject: [PATCH 123/261] feat: introduce runtime store (#157)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/lib.rs         |   1 +
 foyer-common/src/runtime.rs     |  54 ++++++++++
 foyer-storage-bench/src/main.rs |   6 +-
 foyer-storage/src/generic.rs    | 177 ++++++++++++++++++++------------
 foyer-storage/src/lazy.rs       | 173 ++++++++++++++++++++++++-------
 foyer-storage/src/lib.rs        |   1 +
 foyer-storage/src/reclaimer.rs  |   4 +-
 foyer-storage/src/runtime.rs    | 166 ++++++++++++++++++++++++++++++
 foyer-storage/src/storage.rs    |  13 ++-
 foyer-storage/src/store.rs      |  71 +++++--------
 10 files changed, 507 insertions(+), 159 deletions(-)
 create mode 100644 foyer-common/src/runtime.rs
 create mode 100644 foyer-storage/src/runtime.rs

diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 4e3ec27a..3e46aefc 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -24,3 +24,4 @@ pub mod queue;
 pub mod rate;
 pub mod rated_random;
 pub mod rated_ticket;
+pub mod runtime;
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
new file mode 100644
index 00000000..c299f60a
--- /dev/null
+++ b/foyer-common/src/runtime.rs
@@ -0,0 +1,54 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    mem::ManuallyDrop,
+    ops::{Deref, DerefMut},
+};
+
+use tokio::runtime::Runtime;
+
+/// A wrapper around [`Runtime`] that shuts down the runtime in the background when dropped.
+///
+/// This is necessary because directly dropping a nested runtime is not allowed in a parent runtime.
+#[derive(Debug)]
+pub struct BackgroundShutdownRuntime(ManuallyDrop<Runtime>);
+
+impl Drop for BackgroundShutdownRuntime {
+    fn drop(&mut self) {
+        // Safety: The runtime is only dropped once here.
+        let runtime = unsafe { ManuallyDrop::take(&mut self.0) };
+        runtime.shutdown_background();
+    }
+}
+
+impl Deref for BackgroundShutdownRuntime {
+    type Target = Runtime;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for BackgroundShutdownRuntime {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl From<Runtime> for BackgroundShutdownRuntime {
+    fn from(runtime: Runtime) -> Self {
+        Self(ManuallyDrop::new(runtime))
+    }
+}
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index f20784da..edb3d5ca 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -408,7 +408,7 @@ async fn main() {
     println!("\nTotal:\n{}", analysis);
 }
 
-async fn bench(args: Args, store: Arc<TStore>, metrics: Metrics, stop_tx: broadcast::Sender<()>) {
+async fn bench(args: Args, store: TStore, metrics: Metrics, stop_tx: broadcast::Sender<()>) {
     let w_rate = if args.w_rate == 0.0 {
         None
     } else {
@@ -457,7 +457,7 @@ async fn write(
     entry_size_range: Range<usize>,
     rate: Option<f64>,
     index: Arc<AtomicU64>,
-    store: Arc<TStore>,
+    store: TStore,
     time: u64,
     metrics: Metrics,
     mut stop: broadcast::Receiver<()>,
@@ -502,7 +502,7 @@ async fn write(
 async fn read(
     rate: Option<f64>,
     index: Arc<AtomicU64>,
-    store: Arc<TStore>,
+    store: TStore,
     time: u64,
     metrics: Metrics,
     mut stop: broadcast::Receiver<()>,
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index ec46a0a4..c7fa9421 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -134,6 +134,33 @@ where
 
 #[derive(Debug)]
 pub struct GenericStore<K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    inner: Arc<GenericStoreInner<K, V, D, EP, EL>>,
+}
+
+impl<K, V, D, EP, EL> Clone for GenericStore<K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    fn clone(&self) -> Self {
+        Self {
+            inner: Arc::clone(&self.inner),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct GenericStoreInner<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -169,7 +196,7 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    async fn open(config: GenericStoreConfig<K, V, D, EP>) -> Result<Arc<Self>> {
+    async fn open(config: GenericStoreConfig<K, V, D, EP>) -> Result<Self> {
         tracing::info!("open store with config:\n{:#?}", config);
 
         let metrics = Arc::new(METRICS.foyer(&config.name));
@@ -207,7 +234,7 @@ where
             .map(|_| reclaimers_stop_tx.subscribe())
             .collect_vec();
 
-        let store = Arc::new(Self {
+        let inner = GenericStoreInner {
             indices: indices.clone(),
             region_manager: region_manager.clone(),
             device: device.clone(),
@@ -219,13 +246,16 @@ where
             reclaimers_stop_tx,
             metrics: metrics.clone(),
             _marker: PhantomData,
-        });
+        };
+        let store = Self {
+            inner: Arc::new(inner),
+        };
 
-        for admission in store.admissions.iter() {
-            admission.init(&store.indices);
+        for admission in store.inner.admissions.iter() {
+            admission.init(&store.inner.indices);
         }
-        for reinsertion in store.reinsertions.iter() {
-            reinsertion.init(&store.indices);
+        for reinsertion in store.inner.reinsertions.iter() {
+            reinsertion.init(&store.inner.indices);
         }
 
         let flush_rate_limiter = match config.flush_rate_limit {
@@ -274,8 +304,8 @@ where
             .map(|reclaimer| tokio::spawn(async move { reclaimer.run().await.unwrap() }))
             .collect_vec();
 
-        *store.flusher_handles.lock() = flusher_handles;
-        *store.reclaimer_handles.lock() = reclaimer_handles;
+        *store.inner.flusher_handles.lock() = flusher_handles;
+        *store.inner.reclaimer_handles.lock() = reclaimer_handles;
 
         Ok(store)
     }
@@ -285,18 +315,18 @@ where
         self.seal().await;
 
         // stop and wait for reclaimers
-        let handles = self.reclaimer_handles.lock().drain(..).collect_vec();
+        let handles = self.inner.reclaimer_handles.lock().drain(..).collect_vec();
         if !handles.is_empty() {
-            self.reclaimers_stop_tx.send(()).unwrap();
+            self.inner.reclaimers_stop_tx.send(()).unwrap();
         }
         for handle in handles {
             handle.await.unwrap();
         }
 
         // stop and wait for flushers
-        let handles = self.flusher_handles.lock().drain(..).collect_vec();
+        let handles = self.inner.flusher_handles.lock().drain(..).collect_vec();
         if !handles.is_empty() {
-            self.flushers_stop_tx.send(()).unwrap();
+            self.inner.flushers_stop_tx.send(()).unwrap();
         }
         for handle in handles {
             handle.await.unwrap();
@@ -307,31 +337,32 @@ where
 
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[tracing::instrument(skip(self))]
-    fn writer(&self, key: K, weight: usize) -> GenericStoreWriter<'_, K, V, D, EP, EL> {
-        GenericStoreWriter::new(self, key, weight)
+    fn writer(&self, key: K, weight: usize) -> GenericStoreWriter<K, V, D, EP, EL> {
+        GenericStoreWriter::new(self.clone(), key, weight)
     }
 
     #[tracing::instrument(skip(self))]
     fn exists(&self, key: &K) -> Result<bool> {
-        Ok(self.indices.lookup(key).is_some())
+        Ok(self.inner.indices.lookup(key).is_some())
     }
 
     #[tracing::instrument(skip(self))]
     async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
 
-        let index = match self.indices.lookup(key) {
+        let index = match self.inner.indices.lookup(key) {
             Some(index) => index,
             None => {
-                self.metrics
+                self.inner
+                    .metrics
                     .op_duration_lookup_miss
                     .observe(now.elapsed().as_secs_f64());
                 return Ok(None);
             }
         };
 
-        self.region_manager.record_access(&index.region);
-        let region = self.region_manager.region(&index.region);
+        self.inner.region_manager.record_access(&index.region);
+        let region = self.inner.region_manager.region(&index.region);
         let start = index.offset as usize;
         let end = start + index.len as usize;
 
@@ -340,26 +371,31 @@ where
             Some(slice) => slice,
             None => {
                 // Remove index if the storage layer fails to lookup it (because of region version mismatch).
-                self.indices.remove(key);
-                self.metrics
+                self.inner.indices.remove(key);
+                self.inner
+                    .metrics
                     .op_duration_lookup_miss
                     .observe(now.elapsed().as_secs_f64());
                 return Ok(None);
             }
         };
-        self.metrics.op_bytes_lookup.inc_by(slice.len() as u64);
+        self.inner
+            .metrics
+            .op_bytes_lookup
+            .inc_by(slice.len() as u64);
 
         let res = match read_entry::<K, V>(slice.as_ref()) {
             Some((_key, value)) => Ok(Some(value)),
             None => {
                 // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
-                self.indices.remove(key);
+                self.inner.indices.remove(key);
                 Ok(None)
             }
         };
         drop(slice);
 
-        self.metrics
+        self.inner
+            .metrics
             .op_duration_lookup_hit
             .observe(now.elapsed().as_secs_f64());
 
@@ -368,16 +404,16 @@ where
 
     #[tracing::instrument(skip(self))]
     fn remove(&self, key: &K) -> Result<bool> {
-        let _timer = self.metrics.op_duration_remove.start_timer();
+        let _timer = self.inner.metrics.op_duration_remove.start_timer();
 
-        let res = self.indices.remove(key).is_some();
+        let res = self.inner.indices.remove(key).is_some();
 
         Ok(res)
     }
 
     #[tracing::instrument(skip(self))]
     fn clear(&self) -> Result<()> {
-        self.indices.clear();
+        self.inner.indices.clear();
 
         // TODO(MrCroxx): set all regions as clean?
 
@@ -385,21 +421,21 @@ where
     }
 
     pub(crate) fn indices(&self) -> &Arc<Indices<K>> {
-        &self.indices
+        &self.inner.indices
     }
 
     pub(crate) fn reinsertions(&self) -> &Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>> {
-        &self.reinsertions
+        &self.inner.reinsertions
     }
 
     fn serialized_len(&self, key: &K, value: &V) -> usize {
         let unaligned =
             EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len();
-        bits::align_up(self.device.align(), unaligned)
+        bits::align_up(self.inner.device.align(), unaligned)
     }
 
     async fn seal(&self) {
-        self.region_manager.seal().await;
+        self.inner.region_manager.seal().await;
     }
 
     #[tracing::instrument(skip(self))]
@@ -409,11 +445,11 @@ where
         let (tx, rx) = async_channel::bounded(concurrency);
 
         let mut handles = vec![];
-        for region_id in 0..self.device.regions() as RegionId {
+        for region_id in 0..self.inner.device.regions() as RegionId {
             let itx = tx.clone();
             let irx = rx.clone();
-            let region_manager = self.region_manager.clone();
-            let indices = self.indices.clone();
+            let region_manager = self.inner.region_manager.clone();
+            let indices = self.inner.indices.clone();
             let handle = tokio::spawn(async move {
                 itx.send(()).await.unwrap();
                 let res = Self::recover_region(region_id, region_manager, indices).await;
@@ -435,13 +471,14 @@ where
         }
 
         tracing::info!("finish store recovery, {} region recovered", recovered);
-        self.metrics
+        self.inner
+            .metrics
             .total_bytes
-            .set((recovered * self.device.region_size()) as u64);
+            .set((recovered * self.inner.device.region_size()) as u64);
 
         // Force trigger reclamation.
-        if recovered == self.device.regions() {
-            self.region_manager.clean_regions().flash();
+        if recovered == self.inner.device.regions() {
+            self.inner.region_manager.clean_regions().flash();
         }
 
         Ok(())
@@ -467,9 +504,9 @@ where
         Ok(res)
     }
 
-    fn judge_inner(&self, writer: &mut GenericStoreWriter<'_, K, V, D, EP, EL>) {
-        for (index, admission) in self.admissions.iter().enumerate() {
-            let judge = admission.judge(&writer.key, writer.weight, &self.metrics);
+    fn judge_inner(&self, writer: &mut GenericStoreWriter<K, V, D, EP, EL>) {
+        for (index, admission) in self.inner.admissions.iter().enumerate() {
+            let judge = admission.judge(&writer.key, writer.weight, &self.inner.metrics);
             writer.judges.set(index, judge);
         }
         writer.is_judged = true;
@@ -477,7 +514,7 @@ where
 
     async fn apply_writer(
         &self,
-        mut writer: GenericStoreWriter<'_, K, V, D, EP, EL>,
+        mut writer: GenericStoreWriter<K, V, D, EP, EL>,
         value: V,
     ) -> Result<bool> {
         debug_assert!(!writer.is_inserted);
@@ -491,9 +528,9 @@ where
         writer.is_inserted = true;
         let key = &writer.key;
 
-        for (i, admission) in self.admissions.iter().enumerate() {
+        for (i, admission) in self.inner.admissions.iter().enumerate() {
             let judge = writer.judges.get(i);
-            admission.on_insert(key, writer.weight, &self.metrics, judge);
+            admission.on_insert(key, writer.weight, &self.inner.metrics, judge);
         }
 
         let serialized_len = self.serialized_len(key, &value);
@@ -505,9 +542,13 @@ where
             );
         }
 
-        self.metrics.op_bytes_insert.inc_by(serialized_len as u64);
+        self.inner
+            .metrics
+            .op_bytes_insert
+            .inc_by(serialized_len as u64);
 
         let mut slice = match self
+            .inner
             .region_manager
             .allocate(serialized_len, !writer.is_skippable)
             .await
@@ -531,10 +572,11 @@ where
         };
         drop(slice);
 
-        self.indices.insert(index);
+        self.inner.indices.insert(index);
 
         let duration = now.elapsed() + writer.duration;
-        self.metrics
+        self.inner
+            .metrics
             .op_duration_insert_inserted
             .observe(duration.as_secs_f64());
 
@@ -542,7 +584,7 @@ where
     }
 }
 
-pub struct GenericStoreWriter<'a, K, V, D, EP, EL>
+pub struct GenericStoreWriter<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -550,7 +592,7 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    store: &'a GenericStore<K, V, D, EP, EL>,
+    store: GenericStore<K, V, D, EP, EL>,
     key: K,
     weight: usize,
 
@@ -564,7 +606,7 @@ where
     is_skippable: bool,
 }
 
-impl<'a, K, V, D, EP, EL> GenericStoreWriter<'a, K, V, D, EP, EL>
+impl<K, V, D, EP, EL> GenericStoreWriter<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -572,12 +614,13 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    fn new(store: &'a GenericStore<K, V, D, EP, EL>, key: K, weight: usize) -> Self {
+    fn new(store: GenericStore<K, V, D, EP, EL>, key: K, weight: usize) -> Self {
+        let judges = Judges::new(store.inner.admissions.len());
         Self {
             store,
             key,
             weight,
-            judges: Judges::new(store.admissions.len()),
+            judges,
             is_judged: false,
             duration: Duration::from_nanos(0),
             is_inserted: false,
@@ -587,16 +630,18 @@ where
 
     /// Judge if the entry can be admitted by configured admission policies.
     pub fn judge(&mut self) -> bool {
+        let store = self.store.clone();
         if !self.is_judged {
             let now = Instant::now();
-            self.store.judge_inner(self);
+            store.judge_inner(self);
             self.duration = now.elapsed();
         }
         self.judges.judge()
     }
 
     pub async fn finish(self, value: V) -> Result<bool> {
-        self.store.apply_writer(self, value).await
+        let store = self.store.clone();
+        store.apply_writer(self, value).await
     }
 
     pub fn set_force(&mut self) {
@@ -612,7 +657,7 @@ where
     }
 }
 
-impl<'a, K, V, D, EP, EL> Debug for GenericStoreWriter<'a, K, V, D, EP, EL>
+impl<K, V, D, EP, EL> Debug for GenericStoreWriter<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -632,7 +677,7 @@ where
     }
 }
 
-impl<'a, K, V, D, EP, EL> Drop for GenericStoreWriter<'a, K, V, D, EP, EL>
+impl<K, V, D, EP, EL> Drop for GenericStoreWriter<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -643,24 +688,27 @@ where
     fn drop(&mut self) {
         if !self.is_inserted {
             self.store
+                .inner
                 .metrics
                 .op_duration_insert_dropped
                 .observe(self.duration.as_secs_f64());
             let mut filtered = false;
             if self.is_judged {
-                for (i, admission) in self.store.admissions.iter().enumerate() {
+                for (i, admission) in self.store.inner.admissions.iter().enumerate() {
                     let judge = self.judges.get(i);
-                    admission.on_drop(&self.key, self.weight, &self.store.metrics, judge);
+                    admission.on_drop(&self.key, self.weight, &self.store.inner.metrics, judge);
                 }
                 filtered = !self.judge();
             }
             if filtered {
                 self.store
+                    .inner
                     .metrics
                     .op_duration_insert_filtered
                     .observe(self.duration.as_secs_f64());
             } else {
                 self.store
+                    .inner
                     .metrics
                     .op_duration_insert_dropped
                     .observe(self.duration.as_secs_f64());
@@ -906,7 +954,7 @@ where
     }
 }
 
-impl<'a, K, V, D, EP, EL> StorageWriter for GenericStoreWriter<'a, K, V, D, EP, EL>
+impl<K, V, D, EP, EL> StorageWriter for GenericStoreWriter<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -926,7 +974,7 @@ where
     }
 }
 
-impl<'a, K, V, D, EP, EL> ForceStorageWriter for GenericStoreWriter<'a, K, V, D, EP, EL>
+impl<K, V, D, EP, EL> ForceStorageWriter for GenericStoreWriter<K, V, D, EP, EL>
 where
     K: Key,
     V: Value,
@@ -950,10 +998,9 @@ where
     type Key = K;
     type Value = V;
     type Config = GenericStoreConfig<K, V, D, EP>;
-    type Owned = Arc<Self>;
-    type Writer<'a> = GenericStoreWriter<'a, K, V, D, EP, EL>;
+    type Writer = GenericStoreWriter<K, V, D, EP, EL>;
 
-    async fn open(config: Self::Config) -> Result<Self::Owned> {
+    async fn open(config: Self::Config) -> Result<Self> {
         Self::open(config).await
     }
 
@@ -961,7 +1008,7 @@ where
         self.close().await
     }
 
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
         self.writer(key, weight)
     }
 
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 3dc2a391..439277b1 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -12,59 +12,152 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::{Arc, OnceLock};
+use std::{
+    marker::PhantomData,
+    sync::{Arc, OnceLock},
+};
 
 use crate::{
     error::Result,
-    storage::Storage,
-    store::{Store, StoreConfig, StoreWriter},
+    storage::{Storage, StorageWriter},
 };
 use foyer_common::code::{Key, Value};
 use futures::Future;
 
 #[derive(Debug)]
-pub struct LazyStore<K, V>
+pub struct NoneStoreWriter<K: Key, V: Value>(PhantomData<(K, V)>);
+
+impl<K: Key, V: Value> StorageWriter for NoneStoreWriter<K, V> {
+    type Key = K;
+    type Value = V;
+
+    fn judge(&mut self) -> bool {
+        false
+    }
+
+    async fn finish(self, _: Self::Value) -> Result<bool> {
+        Ok(false)
+    }
+}
+
+#[derive(Debug)]
+pub struct NoneStore<K: Key, V: Value>(PhantomData<(K, V)>);
+
+impl<K: Key, V: Value> Clone for NoneStore<K, V> {
+    fn clone(&self) -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<K: Key, V: Value> Storage for NoneStore<K, V> {
+    type Key = K;
+    type Value = V;
+    type Config = ();
+    type Writer = NoneStoreWriter<K, V>;
+
+    #[expect(clippy::let_unit_value)]
+    async fn open(_: Self::Config) -> Result<Self> {
+        Ok(NoneStore(PhantomData))
+    }
+
+    async fn close(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn writer(&self, _: Self::Key, _: usize) -> Self::Writer {
+        NoneStoreWriter(PhantomData)
+    }
+
+    fn exists(&self, _: &Self::Key) -> Result<bool> {
+        Ok(false)
+    }
+
+    async fn lookup(&self, _: &Self::Key) -> Result<Option<Self::Value>> {
+        Ok(None)
+    }
+
+    fn remove(&self, _: &Self::Key) -> Result<bool> {
+        Ok(false)
+    }
+
+    fn clear(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub enum LazyStoreWriter<K, V, S>
 where
     K: Key,
     V: Value,
+    S: Storage<Key = K, Value = V>,
 {
-    once: Arc<OnceLock<Store<K, V>>>,
-    none: Store<K, V>,
+    Store { writer: S::Writer },
+    None { writer: NoneStoreWriter<K, V> },
 }
 
-impl<K, V> Clone for LazyStore<K, V>
+impl<K, V, S> StorageWriter for LazyStoreWriter<K, V, S>
 where
     K: Key,
     V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    type Key = K;
+    type Value = V;
+
+    fn judge(&mut self) -> bool {
+        match self {
+            LazyStoreWriter::Store { writer } => writer.judge(),
+            LazyStoreWriter::None { writer } => writer.judge(),
+        }
+    }
+
+    async fn finish(self, value: Self::Value) -> Result<bool> {
+        match self {
+            LazyStoreWriter::Store { writer } => writer.finish(value).await,
+            LazyStoreWriter::None { writer } => writer.finish(value).await,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct LazyStore<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    once: Arc<OnceLock<S>>,
+    none: NoneStore<K, V>,
+}
+
+impl<K, V, S> Clone for LazyStore<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
 {
     fn clone(&self) -> Self {
         Self {
             once: Arc::clone(&self.once),
-            none: Store::None,
+            none: NoneStore(PhantomData),
         }
     }
 }
 
-impl<K, V> LazyStore<K, V>
+impl<K, V, S> LazyStore<K, V, S>
 where
     K: Key,
     V: Value,
+    S: Storage<Key = K, Value = V>,
 {
-    pub fn lazy(config: StoreConfig<K, V>) -> Self {
-        let (res, task) = Self::lazy_with_task(config);
-        tokio::spawn(task);
-        res
-    }
-
-    pub fn lazy_with_task(
-        config: StoreConfig<K, V>,
-    ) -> (Self, impl Future<Output = Result<Store<K, V>>> + Send) {
+    fn with_task(config: S::Config) -> (Self, impl Future<Output = Result<S>> + Send) {
         let once = Arc::new(OnceLock::new());
 
         let task = {
             let once = once.clone();
             async move {
-                let store = match Store::open(config).await {
+                let store = match S::open(config).await {
                     Ok(store) => store,
                     Err(e) => {
                         tracing::error!("Lazy open store fail: {}", e);
@@ -78,32 +171,28 @@ where
 
         let res = Self {
             once,
-            none: Store::None,
+            none: NoneStore(PhantomData),
         };
 
         (res, task)
     }
 }
 
-impl<K, V> Storage for LazyStore<K, V>
+impl<K, V, S> Storage for LazyStore<K, V, S>
 where
     K: Key,
     V: Value,
+    S: Storage<Key = K, Value = V>,
 {
     type Key = K;
     type Value = V;
-    type Config = StoreConfig<K, V>;
-    type Owned = Self;
-    type Writer<'a> = StoreWriter<'a, K, V>;
+    type Config = S::Config;
+    type Writer = LazyStoreWriter<K, V, S>;
 
-    async fn open(config: Self::Config) -> Result<Self::Owned> {
-        let once = Arc::new(OnceLock::new());
-        let store = Store::open(config).await?;
-        once.set(store).unwrap();
-        Ok(Self {
-            once,
-            none: Store::None,
-        })
+    async fn open(config: S::Config) -> Result<Self> {
+        let (store, task) = Self::with_task(config);
+        tokio::spawn(task);
+        Ok(store)
     }
 
     async fn close(&self) -> Result<()> {
@@ -113,10 +202,14 @@ where
         }
     }
 
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
         match self.once.get() {
-            Some(store) => store.writer(key, weight),
-            None => self.none.writer(key, weight),
+            Some(store) => LazyStoreWriter::Store {
+                writer: store.writer(key, weight),
+            },
+            None => LazyStoreWriter::None {
+                writer: NoneStoreWriter(PhantomData),
+            },
         }
     }
 
@@ -155,7 +248,11 @@ mod tests {
 
     use foyer_intrusive::eviction::fifo::FifoConfig;
 
-    use crate::{device::fs::FsDeviceConfig, storage::StorageExt, store::FifoFsStoreConfig};
+    use crate::{
+        device::fs::FsDeviceConfig,
+        storage::StorageExt,
+        store::{FifoFsStoreConfig, Store},
+    };
 
     use super::*;
 
@@ -189,7 +286,7 @@ mod tests {
             clean_region_threshold: 1,
         };
 
-        let (store, task) = LazyStore::lazy_with_task(config.into());
+        let (store, task) = LazyStore::<_, _, Store<_, _>>::with_task(config.into());
 
         assert!(!store.insert(100, 100).await.unwrap());
 
@@ -224,7 +321,7 @@ mod tests {
             clean_region_threshold: 1,
         };
 
-        let (store, task) = LazyStore::lazy_with_task(config.into());
+        let (store, task) = LazyStore::<_, _, Store<_, _>>::with_task(config.into());
 
         assert!(store.lookup(&100).await.unwrap().is_none());
 
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 54fcd6f9..cd014c3b 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -37,6 +37,7 @@ pub mod reclaimer;
 pub mod region;
 pub mod region_manager;
 pub mod reinsertion;
+pub mod runtime;
 pub mod slice;
 pub mod storage;
 pub mod store;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 4c9a4b86..6507e2d5 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -42,7 +42,7 @@ where
 {
     threshold: usize,
 
-    store: Arc<GenericStore<K, V, D, EP, EL>>,
+    store: GenericStore<K, V, D, EP, EL>,
 
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
@@ -63,7 +63,7 @@ where
 {
     pub fn new(
         threshold: usize,
-        store: Arc<GenericStore<K, V, D, EP, EL>>,
+        store: GenericStore<K, V, D, EP, EL>,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         rate_limiter: Option<Arc<RateLimiter>>,
         metrics: Arc<Metrics>,
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
new file mode 100644
index 00000000..66a39afe
--- /dev/null
+++ b/foyer-storage/src/runtime.rs
@@ -0,0 +1,166 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::Arc;
+
+use foyer_common::{
+    code::{Key, Value},
+    runtime::BackgroundShutdownRuntime,
+};
+
+use crate::{
+    error::Result,
+    storage::{Storage, StorageWriter},
+};
+
+#[derive(Debug)]
+pub struct RuntimeConfig {
+    pub worker_threads: Option<usize>,
+    pub thread_name: Option<String>,
+}
+
+#[derive(Debug)]
+pub struct RuntimeStoreConfig<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    pub store: S::Config,
+    pub runtime: RuntimeConfig,
+}
+
+#[derive(Debug)]
+pub struct RuntimeStoreWriter<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    runtime: Arc<BackgroundShutdownRuntime>,
+    writer: S::Writer,
+}
+
+impl<K, V, S> StorageWriter for RuntimeStoreWriter<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    type Key = K;
+    type Value = V;
+
+    fn judge(&mut self) -> bool {
+        self.writer.judge()
+    }
+
+    async fn finish(self, value: Self::Value) -> Result<bool> {
+        self.runtime
+            .spawn(async move { self.writer.finish(value).await })
+            .await
+            .unwrap()
+    }
+}
+
+#[derive(Debug)]
+pub struct RuntimeStore<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    runtime: Arc<BackgroundShutdownRuntime>,
+    store: S,
+}
+
+impl<K, V, S> Clone for RuntimeStore<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    fn clone(&self) -> Self {
+        Self {
+            runtime: Arc::clone(&self.runtime),
+            store: self.store.clone(),
+        }
+    }
+}
+
+impl<K, V, S> Storage for RuntimeStore<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    type Key = K;
+    type Value = V;
+    type Config = RuntimeStoreConfig<K, V, S>;
+    type Writer = RuntimeStoreWriter<K, V, S>;
+
+    async fn open(config: Self::Config) -> Result<Self> {
+        let mut builder = tokio::runtime::Builder::new_multi_thread();
+        if let Some(worker_threads) = config.runtime.worker_threads {
+            builder.worker_threads(worker_threads);
+        }
+        if let Some(thread_name) = config.runtime.thread_name {
+            builder.thread_name(thread_name);
+        }
+        let runtime = builder.enable_all().build().map_err(anyhow::Error::from)?;
+        let runtime = BackgroundShutdownRuntime::from(runtime);
+        let runtime = Arc::new(runtime);
+        let store = runtime
+            .spawn(async move { S::open(config.store).await })
+            .await
+            .unwrap()?;
+        Ok(Self { runtime, store })
+    }
+
+    async fn close(&self) -> Result<()> {
+        let store = self.store.clone();
+        self.runtime
+            .spawn(async move { store.close().await })
+            .await
+            .unwrap()
+    }
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
+        let writer = self.store.writer(key, weight);
+        RuntimeStoreWriter {
+            runtime: self.runtime.clone(),
+            writer,
+        }
+    }
+
+    fn exists(&self, key: &Self::Key) -> crate::error::Result<bool> {
+        self.store.exists(key)
+    }
+
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        let store = self.store.clone();
+        let key = key.clone();
+        self.runtime
+            .spawn(async move { store.lookup(&key).await })
+            .await
+            .unwrap()
+    }
+
+    fn remove(&self, key: &Self::Key) -> crate::error::Result<bool> {
+        self.store.remove(key)
+    }
+
+    fn clear(&self) -> crate::error::Result<()> {
+        self.store.clear()
+    }
+}
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 7da016cc..0e2715ed 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -30,20 +30,19 @@ pub trait StorageWriter: Send + Sync + Debug {
     fn finish(self, value: Self::Value) -> impl Future<Output = Result<bool>> + Send;
 }
 
-pub trait Storage: Send + Sync + Debug + 'static {
+pub trait Storage: Send + Sync + Debug + Clone + 'static {
     type Key: Key;
     type Value: Value;
     type Config: Send + Debug;
-    type Owned: Send + Sync + Debug + 'static;
-    type Writer<'a>: StorageWriter<Key = Self::Key, Value = Self::Value>;
+    type Writer: StorageWriter<Key = Self::Key, Value = Self::Value>;
 
     #[must_use]
-    fn open(config: Self::Config) -> impl Future<Output = Result<Self::Owned>> + Send;
+    fn open(config: Self::Config) -> impl Future<Output = Result<Self>> + Send;
 
     #[must_use]
     fn close(&self) -> impl Future<Output = Result<()>> + Send;
 
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_>;
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer;
 
     fn exists(&self, key: &Self::Key) -> Result<bool>;
 
@@ -194,7 +193,7 @@ pub trait ForceStorageWriter: StorageWriter {
 
 pub trait ForceStorageExt: Storage
 where
-    for<'w> Self::Writer<'w>: ForceStorageWriter,
+    Self::Writer: ForceStorageWriter,
 {
     #[tracing::instrument(skip(self, value))]
     fn insert_force(
@@ -281,6 +280,6 @@ where
 impl<S> ForceStorageExt for S
 where
     S: Storage,
-    for<'w> S::Writer<'w>: ForceStorageWriter,
+    S::Writer: ForceStorageWriter,
 {
 }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 6e90514c..c22b42a0 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -12,8 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::Arc;
-
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::eviction::{
     fifo::{Fifo, FifoLink},
@@ -35,8 +33,8 @@ pub type LruFsStore<K, V> =
 pub type LruFsStoreConfig<K, V> =
     GenericStoreConfig<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>>;
 
-pub type LruFsStoreWriter<'w, K, V> =
-    GenericStoreWriter<'w, K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
+pub type LruFsStoreWriter<K, V> =
+    GenericStoreWriter<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
 
 pub type LfuFsStore<K, V> =
     GenericStore<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
@@ -44,8 +42,8 @@ pub type LfuFsStore<K, V> =
 pub type LfuFsStoreConfig<K, V> =
     GenericStoreConfig<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>>;
 
-pub type LfuFsStoreWriter<'w, K, V> =
-    GenericStoreWriter<'w, K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
+pub type LfuFsStoreWriter<K, V> =
+    GenericStoreWriter<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
 
 pub type FifoFsStore<K, V> =
     GenericStore<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
@@ -53,8 +51,8 @@ pub type FifoFsStore<K, V> =
 pub type FifoFsStoreConfig<K, V> =
     GenericStoreConfig<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
-pub type FifoFsStoreWriter<'w, K, V> =
-    GenericStoreWriter<'w, K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
+pub type FifoFsStoreWriter<K, V> =
+    GenericStoreWriter<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
 #[derive(Debug)]
 pub enum StoreConfig<K, V>
@@ -65,7 +63,6 @@ where
     LruFsStoreConfig { config: LruFsStoreConfig<K, V> },
     LfuFsStoreConfig { config: LfuFsStoreConfig<K, V> },
     FifoFsStoreConfig { config: FifoFsStoreConfig<K, V> },
-    None,
 }
 
 impl<K, V> From<LruFsStoreConfig<K, V>> for StoreConfig<K, V>
@@ -99,43 +96,42 @@ where
 }
 
 #[derive(Debug)]
-pub enum StoreWriter<'a, K, V>
+pub enum StoreWriter<K, V>
 where
     K: Key,
     V: Value,
 {
-    LruFsStorWriter { writer: LruFsStoreWriter<'a, K, V> },
-    LfuFsStorWriter { writer: LfuFsStoreWriter<'a, K, V> },
-    FifoFsStoreWriter { writer: FifoFsStoreWriter<'a, K, V> },
-    None,
+    LruFsStorWriter { writer: LruFsStoreWriter<K, V> },
+    LfuFsStorWriter { writer: LfuFsStoreWriter<K, V> },
+    FifoFsStoreWriter { writer: FifoFsStoreWriter<K, V> },
 }
 
-impl<'a, K, V> From<LruFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
+impl<K, V> From<LruFsStoreWriter<K, V>> for StoreWriter<K, V>
 where
     K: Key,
     V: Value,
 {
-    fn from(writer: LruFsStoreWriter<'a, K, V>) -> Self {
+    fn from(writer: LruFsStoreWriter<K, V>) -> Self {
         StoreWriter::LruFsStorWriter { writer }
     }
 }
 
-impl<'a, K, V> From<LfuFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
+impl<K, V> From<LfuFsStoreWriter<K, V>> for StoreWriter<K, V>
 where
     K: Key,
     V: Value,
 {
-    fn from(writer: LfuFsStoreWriter<'a, K, V>) -> Self {
+    fn from(writer: LfuFsStoreWriter<K, V>) -> Self {
         StoreWriter::LfuFsStorWriter { writer }
     }
 }
 
-impl<'a, K, V> From<FifoFsStoreWriter<'a, K, V>> for StoreWriter<'a, K, V>
+impl<K, V> From<FifoFsStoreWriter<K, V>> for StoreWriter<K, V>
 where
     K: Key,
     V: Value,
 {
-    fn from(writer: FifoFsStoreWriter<'a, K, V>) -> Self {
+    fn from(writer: FifoFsStoreWriter<K, V>) -> Self {
         StoreWriter::FifoFsStoreWriter { writer }
     }
 }
@@ -146,10 +142,9 @@ where
     K: Key,
     V: Value,
 {
-    LruFsStore { store: Arc<LruFsStore<K, V>> },
-    LfuFsStore { store: Arc<LfuFsStore<K, V>> },
-    FifoFsStore { store: Arc<FifoFsStore<K, V>> },
-    None,
+    LruFsStore { store: LruFsStore<K, V> },
+    LfuFsStore { store: LfuFsStore<K, V> },
+    FifoFsStore { store: FifoFsStore<K, V> },
 }
 
 impl<K, V> Clone for Store<K, V>
@@ -160,20 +155,19 @@ where
     fn clone(&self) -> Self {
         match self {
             Self::LruFsStore { store } => Self::LruFsStore {
-                store: Arc::clone(store),
+                store: store.clone(),
             },
             Self::LfuFsStore { store } => Self::LfuFsStore {
-                store: Arc::clone(store),
+                store: store.clone(),
             },
             Self::FifoFsStore { store } => Self::FifoFsStore {
-                store: Arc::clone(store),
+                store: store.clone(),
             },
-            Self::None => Self::None,
         }
     }
 }
 
-impl<'a, K, V> StorageWriter for StoreWriter<'a, K, V>
+impl<K, V> StorageWriter for StoreWriter<K, V>
 where
     K: Key,
     V: Value,
@@ -186,7 +180,6 @@ where
             StoreWriter::LruFsStorWriter { writer } => writer.judge(),
             StoreWriter::LfuFsStorWriter { writer } => writer.judge(),
             StoreWriter::FifoFsStoreWriter { writer } => writer.judge(),
-            StoreWriter::None => false,
         }
     }
 
@@ -195,12 +188,11 @@ where
             StoreWriter::LruFsStorWriter { writer } => writer.finish(value).await,
             StoreWriter::LfuFsStorWriter { writer } => writer.finish(value).await,
             StoreWriter::FifoFsStoreWriter { writer } => writer.finish(value).await,
-            StoreWriter::None => Ok(false),
         }
     }
 }
 
-impl<'a, K, V> ForceStorageWriter for StoreWriter<'a, K, V>
+impl<K, V> ForceStorageWriter for StoreWriter<K, V>
 where
     K: Key,
     V: Value,
@@ -210,7 +202,6 @@ where
             StoreWriter::LruFsStorWriter { writer } => writer.set_force(),
             StoreWriter::LfuFsStorWriter { writer } => writer.set_force(),
             StoreWriter::FifoFsStoreWriter { writer } => writer.set_force(),
-            StoreWriter::None => {}
         }
     }
 }
@@ -223,10 +214,9 @@ where
     type Key = K;
     type Value = V;
     type Config = StoreConfig<K, V>;
-    type Owned = Self;
-    type Writer<'a> = StoreWriter<'a, K, V>;
+    type Writer = StoreWriter<K, V>;
 
-    async fn open(config: Self::Config) -> Result<Self::Owned> {
+    async fn open(config: Self::Config) -> Result<Self> {
         match config {
             StoreConfig::LruFsStoreConfig { config } => {
                 let store = LruFsStore::open(config).await?;
@@ -240,7 +230,6 @@ where
                 let store = FifoFsStore::open(config).await?;
                 Ok(Self::FifoFsStore { store })
             }
-            StoreConfig::None => Ok(Self::None),
         }
     }
 
@@ -249,16 +238,14 @@ where
             Store::LruFsStore { store } => store.close().await,
             Store::LfuFsStore { store } => store.close().await,
             Store::FifoFsStore { store } => store.close().await,
-            Store::None => Ok(()),
         }
     }
 
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer<'_> {
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
         match self {
             Store::LruFsStore { store } => store.writer(key, weight).into(),
             Store::LfuFsStore { store } => store.writer(key, weight).into(),
             Store::FifoFsStore { store } => store.writer(key, weight).into(),
-            Store::None => StoreWriter::None,
         }
     }
 
@@ -267,7 +254,6 @@ where
             Store::LruFsStore { store } => store.exists(key),
             Store::LfuFsStore { store } => store.exists(key),
             Store::FifoFsStore { store } => store.exists(key),
-            Store::None => Ok(false),
         }
     }
 
@@ -276,7 +262,6 @@ where
             Store::LruFsStore { store } => store.lookup(key).await,
             Store::LfuFsStore { store } => store.lookup(key).await,
             Store::FifoFsStore { store } => store.lookup(key).await,
-            Store::None => Ok(None),
         }
     }
 
@@ -285,7 +270,6 @@ where
             Store::LruFsStore { store } => store.remove(key),
             Store::LfuFsStore { store } => store.remove(key),
             Store::FifoFsStore { store } => store.remove(key),
-            Store::None => Ok(false),
         }
     }
 
@@ -294,7 +278,6 @@ where
             Store::LruFsStore { store } => store.clear(),
             Store::LfuFsStore { store } => store.clear(),
             Store::FifoFsStore { store } => store.clear(),
-            Store::None => Ok(()),
         }
     }
 }

From 5127202229b66f817b4bff8b7bf7dd9037dc9969 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 8 Oct 2023 05:30:39 -0500
Subject: [PATCH 124/261] test: introduce basic generic storage test (#158)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/device/fs.rs      |  12 +-
 foyer-storage/src/device/mod.rs     |   2 +-
 foyer-storage/src/generic.rs        | 143 +++++--------------
 foyer-storage/src/lazy.rs           |   4 +
 foyer-storage/src/lib.rs            |   2 +
 foyer-storage/src/runtime.rs        |  16 ++-
 foyer-storage/src/storage.rs        |   6 +-
 foyer-storage/src/store.rs          |  20 +++
 foyer-storage/src/test_utils.rs     | 127 +++++++++++++++++
 foyer-storage/tests/storage_test.rs | 213 ++++++++++++++++++++++++++++
 10 files changed, 429 insertions(+), 116 deletions(-)
 create mode 100644 foyer-storage/src/test_utils.rs
 create mode 100644 foyer-storage/tests/storage_test.rs

diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 32c03d56..539394ee 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -88,7 +88,11 @@ impl Device for FsDevice {
         offset: u64,
         len: usize,
     ) -> DeviceResult<usize> {
-        assert!(offset as usize + len <= self.inner.config.file_capacity);
+        let file_capacity = self.inner.config.file_capacity;
+        assert!(
+            offset as usize + len <= file_capacity,
+            "offset ({offset}) + len ({len}) <= file capacity ({file_capacity})"
+        );
 
         let fd = self.fd(region);
 
@@ -109,7 +113,11 @@ impl Device for FsDevice {
         offset: u64,
         len: usize,
     ) -> DeviceResult<usize> {
-        assert!(offset as usize + len <= self.inner.config.file_capacity);
+        let file_capacity = self.inner.config.file_capacity;
+        assert!(
+            offset as usize + len <= file_capacity,
+            "offset ({offset}) + len ({len}) <= file capacity ({file_capacity})"
+        );
 
         let fd = self.fd(region);
 
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 91a50008..a1fb1429 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -28,7 +28,7 @@ pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
 
 pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     type IoBufferAllocator: BufferAllocator;
-    type Config: Send + Debug;
+    type Config: Send + Debug + Clone;
 
     #[must_use]
     fn open(config: Self::Config) -> impl Future<Output = DeviceResult<Self>> + Send;
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index c7fa9421..75f8de22 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -132,6 +132,33 @@ where
     }
 }
 
+impl<K, V, D, EP> Clone for GenericStoreConfig<K, V, D, EP>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy,
+{
+    fn clone(&self) -> Self {
+        Self {
+            name: self.name.clone(),
+            eviction_config: self.eviction_config.clone(),
+            device_config: self.device_config.clone(),
+            allocator_bits: self.allocator_bits,
+            admissions: self.admissions.clone(),
+            reinsertions: self.reinsertions.clone(),
+            buffer_pool_size: self.buffer_pool_size,
+            flushers: self.flushers,
+            flush_rate_limit: self.flush_rate_limit,
+            reclaimers: self.reclaimers,
+            reclaim_rate_limit: self.reclaim_rate_limit,
+            allocation_timeout: self.allocation_timeout,
+            clean_region_threshold: self.clean_region_threshold,
+            recover_concurrency: self.recover_concurrency,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct GenericStore<K, V, D, EP, EL>
 where
@@ -1030,14 +1057,15 @@ where
 }
 
 #[cfg(test)]
-pub mod tests {
-    use std::{collections::HashSet, path::PathBuf};
+mod tests {
+    use std::path::PathBuf;
 
     use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
 
     use crate::{
         device::fs::{FsDevice, FsDeviceConfig},
         storage::StorageExt,
+        test_utils::JudgeRecorder,
     };
 
     use super::*;
@@ -1048,113 +1076,6 @@ pub mod tests {
     type TestStoreConfig =
         GenericStoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
-    #[derive(Debug, Clone)]
-    enum Record<K: Key> {
-        Admit(K),
-        Evict(K),
-    }
-
-    #[derive(Debug)]
-    struct JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        records: Mutex<Vec<Record<K>>>,
-        _marker: PhantomData<V>,
-    }
-
-    impl<K, V> JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        fn dump(&self) -> Vec<Record<K>> {
-            self.records.lock().clone()
-        }
-
-        fn remains(&self) -> HashSet<K> {
-            let records = self.dump();
-            let mut res = HashSet::default();
-            for record in records {
-                match record {
-                    Record::Admit(key) => {
-                        res.insert(key);
-                    }
-                    Record::Evict(key) => {
-                        res.remove(&key);
-                    }
-                }
-            }
-            res
-        }
-    }
-
-    impl<K, V> Default for JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        fn default() -> Self {
-            Self {
-                records: Mutex::new(Vec::default()),
-                _marker: PhantomData,
-            }
-        }
-    }
-
-    impl<K, V> AdmissionPolicy for JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        type Key = K;
-
-        type Value = V;
-
-        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
-            self.records.lock().push(Record::Admit(key.clone()));
-            true
-        }
-
-        fn on_insert(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
-
-        fn on_drop(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
-    }
-
-    impl<K, V> ReinsertionPolicy for JudgeRecorder<K, V>
-    where
-        K: Key,
-        V: Value,
-    {
-        type Key = K;
-
-        type Value = V;
-
-        fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
-            self.records.lock().push(Record::Evict(key.clone()));
-            false
-        }
-
-        fn on_insert(
-            &self,
-            _key: &Self::Key,
-            _weight: usize,
-            _metrics: &Arc<crate::metrics::Metrics>,
-            _judge: bool,
-        ) {
-        }
-
-        fn on_drop(
-            &self,
-            _key: &Self::Key,
-            _weight: usize,
-            _metrics: &Arc<crate::metrics::Metrics>,
-            _judge: bool,
-        ) {
-        }
-    }
-
     #[tokio::test]
     #[expect(clippy::identity_op)]
     async fn test_recovery() {
@@ -1176,8 +1097,8 @@ pub mod tests {
                 dir: PathBuf::from(tempdir.path()),
                 capacity: 16 * MB,
                 file_capacity: 4 * MB,
-                align: 4096,
-                io_size: 4096 * KB,
+                align: 4 * KB,
+                io_size: 4 * KB,
             },
             allocator_bits: 1,
             admissions,
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 439277b1..5addac3e 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -195,6 +195,10 @@ where
         Ok(store)
     }
 
+    fn is_ready(&self) -> bool {
+        self.once.get().is_some()
+    }
+
     async fn close(&self) -> Result<()> {
         match self.once.get() {
             Some(store) => store.close().await,
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index cd014c3b..2e05bcc1 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -41,3 +41,5 @@ pub mod runtime;
 pub mod slice;
 pub mod storage;
 pub mod store;
+
+pub mod test_utils;
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 66a39afe..5ecb5bc0 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -24,7 +24,7 @@ use crate::{
     storage::{Storage, StorageWriter},
 };
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct RuntimeConfig {
     pub worker_threads: Option<usize>,
     pub thread_name: Option<String>,
@@ -41,6 +41,20 @@ where
     pub runtime: RuntimeConfig,
 }
 
+impl<K, V, S> Clone for RuntimeStoreConfig<K, V, S>
+where
+    K: Key,
+    V: Value,
+    S: Storage<Key = K, Value = V>,
+{
+    fn clone(&self) -> Self {
+        Self {
+            store: self.store.clone(),
+            runtime: self.runtime.clone(),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct RuntimeStoreWriter<K, V, S>
 where
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 0e2715ed..eff65695 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -33,12 +33,16 @@ pub trait StorageWriter: Send + Sync + Debug {
 pub trait Storage: Send + Sync + Debug + Clone + 'static {
     type Key: Key;
     type Value: Value;
-    type Config: Send + Debug;
+    type Config: Send + Clone + Debug;
     type Writer: StorageWriter<Key = Self::Key, Value = Self::Value>;
 
     #[must_use]
     fn open(config: Self::Config) -> impl Future<Output = Result<Self>> + Send;
 
+    fn is_ready(&self) -> bool {
+        true
+    }
+
     #[must_use]
     fn close(&self) -> impl Future<Output = Result<()>> + Send;
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index c22b42a0..57b78805 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -65,6 +65,26 @@ where
     FifoFsStoreConfig { config: FifoFsStoreConfig<K, V> },
 }
 
+impl<K, V> Clone for StoreConfig<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn clone(&self) -> Self {
+        match self {
+            Self::LruFsStoreConfig { config } => Self::LruFsStoreConfig {
+                config: config.clone(),
+            },
+            Self::LfuFsStoreConfig { config } => Self::LfuFsStoreConfig {
+                config: config.clone(),
+            },
+            Self::FifoFsStoreConfig { config } => Self::FifoFsStoreConfig {
+                config: config.clone(),
+            },
+        }
+    }
+}
+
 impl<K, V> From<LruFsStoreConfig<K, V>> for StoreConfig<K, V>
 where
     K: Key,
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
new file mode 100644
index 00000000..5984c6ce
--- /dev/null
+++ b/foyer-storage/src/test_utils.rs
@@ -0,0 +1,127 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.s
+
+use std::{collections::HashSet, marker::PhantomData, sync::Arc};
+
+use foyer_common::code::{Key, Value};
+use parking_lot::Mutex;
+
+use crate::{admission::AdmissionPolicy, metrics::Metrics, reinsertion::ReinsertionPolicy};
+
+#[derive(Debug, Clone)]
+pub enum Record<K: Key> {
+    Admit(K),
+    Evict(K),
+}
+
+#[derive(Debug)]
+pub struct JudgeRecorder<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    records: Mutex<Vec<Record<K>>>,
+    _marker: PhantomData<V>,
+}
+
+impl<K, V> JudgeRecorder<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn dump(&self) -> Vec<Record<K>> {
+        self.records.lock().clone()
+    }
+
+    pub fn remains(&self) -> HashSet<K> {
+        let records = self.dump();
+        let mut res = HashSet::default();
+        for record in records {
+            match record {
+                Record::Admit(key) => {
+                    res.insert(key);
+                }
+                Record::Evict(key) => {
+                    res.remove(&key);
+                }
+            }
+        }
+        res
+    }
+}
+
+impl<K, V> Default for JudgeRecorder<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn default() -> Self {
+        Self {
+            records: Mutex::new(Vec::default()),
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<K, V> AdmissionPolicy for JudgeRecorder<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.records.lock().push(Record::Admit(key.clone()));
+        true
+    }
+
+    fn on_insert(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+
+    fn on_drop(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+}
+
+impl<K, V> ReinsertionPolicy for JudgeRecorder<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+
+    type Value = V;
+
+    fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+        self.records.lock().push(Record::Evict(key.clone()));
+        false
+    }
+
+    fn on_insert(
+        &self,
+        _key: &Self::Key,
+        _weight: usize,
+        _metrics: &Arc<crate::metrics::Metrics>,
+        _judge: bool,
+    ) {
+    }
+
+    fn on_drop(
+        &self,
+        _key: &Self::Key,
+        _weight: usize,
+        _metrics: &Arc<crate::metrics::Metrics>,
+        _judge: bool,
+    ) {
+    }
+}
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
new file mode 100644
index 00000000..6e986eac
--- /dev/null
+++ b/foyer-storage/tests/storage_test.rs
@@ -0,0 +1,213 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(lint_reasons)]
+#![expect(clippy::identity_op)]
+
+use std::{path::PathBuf, sync::Arc, time::Duration};
+
+use foyer_intrusive::eviction::fifo::FifoConfig;
+use foyer_storage::{
+    device::fs::FsDeviceConfig,
+    lazy::LazyStore,
+    runtime::{RuntimeConfig, RuntimeStore, RuntimeStoreConfig},
+    storage::{Storage, StorageExt},
+    store::{FifoFsStoreConfig, Store},
+    test_utils::JudgeRecorder,
+};
+
+const KB: usize = 1024;
+const MB: usize = 1024 * 1024;
+
+const INSERTS: usize = 100;
+const LOOPS: usize = 10;
+
+async fn test_storage<S>(config: S::Config, recorder: Arc<JudgeRecorder<u64, Vec<u8>>>)
+where
+    S: Storage<Key = u64, Value = Vec<u8>>,
+{
+    let store = S::open(config.clone()).await.unwrap();
+    while !store.is_ready() {
+        tokio::time::sleep(Duration::from_millis(10)).await;
+    }
+
+    let mut index = 0;
+
+    for _ in 0..INSERTS as u64 {
+        index += 1;
+        store
+            .insert(index, vec![index as u8; 1 * KB])
+            .await
+            .unwrap();
+    }
+
+    store.close().await.unwrap();
+
+    let remains = recorder.remains();
+
+    for i in 0..INSERTS as u64 * (LOOPS + 1) as u64 {
+        if remains.contains(&i) {
+            assert_eq!(
+                store.lookup(&i).await.unwrap().unwrap(),
+                vec![i as u8; 1 * KB],
+            );
+        } else {
+            assert!(store.lookup(&i).await.unwrap().is_none());
+        }
+    }
+
+    drop(store);
+
+    for _ in 0..LOOPS {
+        let store = S::open(config.clone()).await.unwrap();
+        while !store.is_ready() {
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+
+        let remains = recorder.remains();
+
+        for i in 0..INSERTS as u64 * (LOOPS + 1) as u64 {
+            if remains.contains(&i) {
+                assert_eq!(
+                    store.lookup(&i).await.unwrap().unwrap(),
+                    vec![i as u8; 1 * KB],
+                );
+            } else {
+                assert!(store.lookup(&i).await.unwrap().is_none());
+            }
+        }
+
+        for _ in 0..INSERTS as u64 {
+            index += 1;
+            store
+                .insert(index, vec![index as u8; 1 * KB])
+                .await
+                .unwrap();
+        }
+
+        store.close().await.unwrap();
+
+        let remains = recorder.remains();
+
+        for i in 0..INSERTS as u64 * (LOOPS + 1) as u64 {
+            if remains.contains(&i) {
+                assert_eq!(
+                    store.lookup(&i).await.unwrap().unwrap(),
+                    vec![i as u8; 1 * KB],
+                );
+            } else {
+                assert!(store.lookup(&i).await.unwrap().is_none());
+            }
+        }
+
+        drop(store);
+    }
+}
+
+#[tokio::test]
+async fn test_store() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let recorder = Arc::new(JudgeRecorder::default());
+    let config = FifoFsStoreConfig {
+        name: "".to_string(),
+        eviction_config: FifoConfig,
+        device_config: FsDeviceConfig {
+            dir: PathBuf::from(tempdir.path()),
+            capacity: 4 * MB,
+            file_capacity: 1 * MB,
+            align: 4 * KB,
+            io_size: 4 * KB,
+        },
+        allocator_bits: 0,
+        admissions: vec![recorder.clone()],
+        reinsertions: vec![recorder.clone()],
+        buffer_pool_size: 2 * MB,
+        flushers: 1,
+        flush_rate_limit: 0,
+        reclaimers: 1,
+        reclaim_rate_limit: 0,
+        allocation_timeout: Duration::from_millis(10),
+        clean_region_threshold: 1,
+        recover_concurrency: 2,
+    };
+
+    test_storage::<Store<_, _>>(config.into(), recorder).await;
+}
+
+#[tokio::test]
+async fn test_lazy_store() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let recorder = Arc::new(JudgeRecorder::default());
+    let config = FifoFsStoreConfig {
+        name: "".to_string(),
+        eviction_config: FifoConfig,
+        device_config: FsDeviceConfig {
+            dir: PathBuf::from(tempdir.path()),
+            capacity: 4 * MB,
+            file_capacity: 1 * MB,
+            align: 4 * KB,
+            io_size: 4 * KB,
+        },
+        allocator_bits: 0,
+        admissions: vec![recorder.clone()],
+        reinsertions: vec![recorder.clone()],
+        buffer_pool_size: 2 * MB,
+        flushers: 1,
+        flush_rate_limit: 0,
+        reclaimers: 1,
+        reclaim_rate_limit: 0,
+        allocation_timeout: Duration::from_millis(10),
+        clean_region_threshold: 1,
+        recover_concurrency: 2,
+    };
+
+    test_storage::<LazyStore<_, _, Store<_, _>>>(config.into(), recorder).await;
+}
+
+#[tokio::test]
+async fn test_runtime_store() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let recorder = Arc::new(JudgeRecorder::default());
+    let config = RuntimeStoreConfig {
+        store: FifoFsStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 4 * MB,
+                file_capacity: 1 * MB,
+                align: 4 * KB,
+                io_size: 4 * KB,
+            },
+            allocator_bits: 0,
+            admissions: vec![recorder.clone()],
+            reinsertions: vec![recorder.clone()],
+            buffer_pool_size: 2 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 1,
+            reclaim_rate_limit: 0,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+            recover_concurrency: 2,
+        }
+        .into(),
+        runtime: RuntimeConfig {
+            worker_threads: None,
+            thread_name: None,
+        },
+    };
+
+    test_storage::<RuntimeStore<_, _, Store<_, _>>>(config, recorder).await;
+}

From f85654c55f910f10b2a4c10b6e8e87fb18b683fa Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 8 Oct 2023 08:10:44 -0500
Subject: [PATCH 125/261] chore: export LazyStore and RuntimeStore to simplify
 code (#159)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/lazy.rs           | 34 ++++++++++++++++-------------
 foyer-storage/src/runtime.rs        | 25 ++++++++++++---------
 foyer-storage/tests/storage_test.rs |  8 +++----
 3 files changed, 38 insertions(+), 29 deletions(-)

diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 5addac3e..96614e3d 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -20,6 +20,7 @@ use std::{
 use crate::{
     error::Result,
     storage::{Storage, StorageWriter},
+    store::Store,
 };
 use foyer_common::code::{Key, Value};
 use futures::Future;
@@ -86,7 +87,7 @@ impl<K: Key, V: Value> Storage for NoneStore<K, V> {
 }
 
 #[derive(Debug)]
-pub enum LazyStoreWriter<K, V, S>
+pub enum LazyStorageWriter<K, V, S>
 where
     K: Key,
     V: Value,
@@ -96,7 +97,7 @@ where
     None { writer: NoneStoreWriter<K, V> },
 }
 
-impl<K, V, S> StorageWriter for LazyStoreWriter<K, V, S>
+impl<K, V, S> StorageWriter for LazyStorageWriter<K, V, S>
 where
     K: Key,
     V: Value,
@@ -107,21 +108,21 @@ where
 
     fn judge(&mut self) -> bool {
         match self {
-            LazyStoreWriter::Store { writer } => writer.judge(),
-            LazyStoreWriter::None { writer } => writer.judge(),
+            LazyStorageWriter::Store { writer } => writer.judge(),
+            LazyStorageWriter::None { writer } => writer.judge(),
         }
     }
 
     async fn finish(self, value: Self::Value) -> Result<bool> {
         match self {
-            LazyStoreWriter::Store { writer } => writer.finish(value).await,
-            LazyStoreWriter::None { writer } => writer.finish(value).await,
+            LazyStorageWriter::Store { writer } => writer.finish(value).await,
+            LazyStorageWriter::None { writer } => writer.finish(value).await,
         }
     }
 }
 
 #[derive(Debug)]
-pub struct LazyStore<K, V, S>
+pub struct LazyStorage<K, V, S>
 where
     K: Key,
     V: Value,
@@ -131,7 +132,7 @@ where
     none: NoneStore<K, V>,
 }
 
-impl<K, V, S> Clone for LazyStore<K, V, S>
+impl<K, V, S> Clone for LazyStorage<K, V, S>
 where
     K: Key,
     V: Value,
@@ -145,7 +146,7 @@ where
     }
 }
 
-impl<K, V, S> LazyStore<K, V, S>
+impl<K, V, S> LazyStorage<K, V, S>
 where
     K: Key,
     V: Value,
@@ -178,7 +179,7 @@ where
     }
 }
 
-impl<K, V, S> Storage for LazyStore<K, V, S>
+impl<K, V, S> Storage for LazyStorage<K, V, S>
 where
     K: Key,
     V: Value,
@@ -187,7 +188,7 @@ where
     type Key = K;
     type Value = V;
     type Config = S::Config;
-    type Writer = LazyStoreWriter<K, V, S>;
+    type Writer = LazyStorageWriter<K, V, S>;
 
     async fn open(config: S::Config) -> Result<Self> {
         let (store, task) = Self::with_task(config);
@@ -208,10 +209,10 @@ where
 
     fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
         match self.once.get() {
-            Some(store) => LazyStoreWriter::Store {
+            Some(store) => LazyStorageWriter::Store {
                 writer: store.writer(key, weight),
             },
-            None => LazyStoreWriter::None {
+            None => LazyStorageWriter::None {
                 writer: NoneStoreWriter(PhantomData),
             },
         }
@@ -246,6 +247,9 @@ where
     }
 }
 
+pub type LazyStore<K, V> = LazyStorage<K, V, Store<K, V>>;
+pub type LazyStoreWriter<K, V> = LazyStorageWriter<K, V, Store<K, V>>;
+
 #[cfg(test)]
 mod tests {
     use std::{path::PathBuf, time::Duration};
@@ -290,7 +294,7 @@ mod tests {
             clean_region_threshold: 1,
         };
 
-        let (store, task) = LazyStore::<_, _, Store<_, _>>::with_task(config.into());
+        let (store, task) = LazyStorage::<_, _, Store<_, _>>::with_task(config.into());
 
         assert!(!store.insert(100, 100).await.unwrap());
 
@@ -325,7 +329,7 @@ mod tests {
             clean_region_threshold: 1,
         };
 
-        let (store, task) = LazyStore::<_, _, Store<_, _>>::with_task(config.into());
+        let (store, task) = LazyStorage::<_, _, Store<_, _>>::with_task(config.into());
 
         assert!(store.lookup(&100).await.unwrap().is_none());
 
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 5ecb5bc0..9e9d68a2 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -22,6 +22,7 @@ use foyer_common::{
 use crate::{
     error::Result,
     storage::{Storage, StorageWriter},
+    store::Store,
 };
 
 #[derive(Debug, Clone)]
@@ -31,7 +32,7 @@ pub struct RuntimeConfig {
 }
 
 #[derive(Debug)]
-pub struct RuntimeStoreConfig<K, V, S>
+pub struct RuntimeStorageConfig<K, V, S>
 where
     K: Key,
     V: Value,
@@ -41,7 +42,7 @@ where
     pub runtime: RuntimeConfig,
 }
 
-impl<K, V, S> Clone for RuntimeStoreConfig<K, V, S>
+impl<K, V, S> Clone for RuntimeStorageConfig<K, V, S>
 where
     K: Key,
     V: Value,
@@ -56,7 +57,7 @@ where
 }
 
 #[derive(Debug)]
-pub struct RuntimeStoreWriter<K, V, S>
+pub struct RuntimeStorageWriter<K, V, S>
 where
     K: Key,
     V: Value,
@@ -66,7 +67,7 @@ where
     writer: S::Writer,
 }
 
-impl<K, V, S> StorageWriter for RuntimeStoreWriter<K, V, S>
+impl<K, V, S> StorageWriter for RuntimeStorageWriter<K, V, S>
 where
     K: Key,
     V: Value,
@@ -88,7 +89,7 @@ where
 }
 
 #[derive(Debug)]
-pub struct RuntimeStore<K, V, S>
+pub struct RuntimeStorage<K, V, S>
 where
     K: Key,
     V: Value,
@@ -98,7 +99,7 @@ where
     store: S,
 }
 
-impl<K, V, S> Clone for RuntimeStore<K, V, S>
+impl<K, V, S> Clone for RuntimeStorage<K, V, S>
 where
     K: Key,
     V: Value,
@@ -112,7 +113,7 @@ where
     }
 }
 
-impl<K, V, S> Storage for RuntimeStore<K, V, S>
+impl<K, V, S> Storage for RuntimeStorage<K, V, S>
 where
     K: Key,
     V: Value,
@@ -120,8 +121,8 @@ where
 {
     type Key = K;
     type Value = V;
-    type Config = RuntimeStoreConfig<K, V, S>;
-    type Writer = RuntimeStoreWriter<K, V, S>;
+    type Config = RuntimeStorageConfig<K, V, S>;
+    type Writer = RuntimeStorageWriter<K, V, S>;
 
     async fn open(config: Self::Config) -> Result<Self> {
         let mut builder = tokio::runtime::Builder::new_multi_thread();
@@ -151,7 +152,7 @@ where
 
     fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
         let writer = self.store.writer(key, weight);
-        RuntimeStoreWriter {
+        RuntimeStorageWriter {
             runtime: self.runtime.clone(),
             writer,
         }
@@ -178,3 +179,7 @@ where
         self.store.clear()
     }
 }
+
+pub type RuntimeStore<K, V> = RuntimeStorage<K, V, Store<K, V>>;
+pub type RuntimeStoreWriter<K, V> = RuntimeStorageWriter<K, V, Store<K, V>>;
+pub type RuntimeStoreConfig<K, V> = RuntimeStorageConfig<K, V, Store<K, V>>;
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 6e986eac..c6b48e73 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -21,7 +21,7 @@ use foyer_intrusive::eviction::fifo::FifoConfig;
 use foyer_storage::{
     device::fs::FsDeviceConfig,
     lazy::LazyStore,
-    runtime::{RuntimeConfig, RuntimeStore, RuntimeStoreConfig},
+    runtime::{RuntimeConfig, RuntimeStorageConfig, RuntimeStore},
     storage::{Storage, StorageExt},
     store::{FifoFsStoreConfig, Store},
     test_utils::JudgeRecorder,
@@ -172,14 +172,14 @@ async fn test_lazy_store() {
         recover_concurrency: 2,
     };
 
-    test_storage::<LazyStore<_, _, Store<_, _>>>(config.into(), recorder).await;
+    test_storage::<LazyStore<_, _>>(config.into(), recorder).await;
 }
 
 #[tokio::test]
 async fn test_runtime_store() {
     let tempdir = tempfile::tempdir().unwrap();
     let recorder = Arc::new(JudgeRecorder::default());
-    let config = RuntimeStoreConfig {
+    let config = RuntimeStorageConfig {
         store: FifoFsStoreConfig {
             name: "".to_string(),
             eviction_config: FifoConfig,
@@ -209,5 +209,5 @@ async fn test_runtime_store() {
         },
     };
 
-    test_storage::<RuntimeStore<_, _, Store<_, _>>>(config, recorder).await;
+    test_storage::<RuntimeStore<_, _>>(config, recorder).await;
 }

From 0b9445b246c27c049cc27bc64db0f7990c3742eb Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 8 Oct 2023 22:20:22 -0500
Subject: [PATCH 126/261] chore: export RuntimeLazyStore and fix fn ready
 (#160)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/generic.rs        |  4 +++
 foyer-storage/src/lazy.rs           | 24 ++++++++++--------
 foyer-storage/src/runtime.rs        |  9 +++++++
 foyer-storage/src/storage.rs        |  4 +--
 foyer-storage/src/store.rs          |  8 ++++++
 foyer-storage/tests/storage_test.rs | 39 ++++++++++++++++++++++++++++-
 6 files changed, 74 insertions(+), 14 deletions(-)

diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 75f8de22..d18149fe 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -1031,6 +1031,10 @@ where
         Self::open(config).await
     }
 
+    fn is_ready(&self) -> bool {
+        true
+    }
+
     async fn close(&self) -> Result<()> {
         self.close().await
     }
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 96614e3d..7d17737e 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -23,7 +23,7 @@ use crate::{
     store::Store,
 };
 use foyer_common::code::{Key, Value};
-use futures::Future;
+use tokio::task::JoinHandle;
 
 #[derive(Debug)]
 pub struct NoneStoreWriter<K: Key, V: Value>(PhantomData<(K, V)>);
@@ -61,6 +61,10 @@ impl<K: Key, V: Value> Storage for NoneStore<K, V> {
         Ok(NoneStore(PhantomData))
     }
 
+    fn is_ready(&self) -> bool {
+        true
+    }
+
     async fn close(&self) -> Result<()> {
         Ok(())
     }
@@ -152,10 +156,10 @@ where
     V: Value,
     S: Storage<Key = K, Value = V>,
 {
-    fn with_task(config: S::Config) -> (Self, impl Future<Output = Result<S>> + Send) {
+    fn with_handle(config: S::Config) -> (Self, JoinHandle<Result<S>>) {
         let once = Arc::new(OnceLock::new());
 
-        let task = {
+        let handle = tokio::spawn({
             let once = once.clone();
             async move {
                 let store = match S::open(config).await {
@@ -168,14 +172,14 @@ where
                 once.set(store.clone()).unwrap();
                 Ok(store)
             }
-        };
+        });
 
         let res = Self {
             once,
             none: NoneStore(PhantomData),
         };
 
-        (res, task)
+        (res, handle)
     }
 }
 
@@ -191,7 +195,7 @@ where
     type Writer = LazyStorageWriter<K, V, S>;
 
     async fn open(config: S::Config) -> Result<Self> {
-        let (store, task) = Self::with_task(config);
+        let (store, task) = Self::with_handle(config);
         tokio::spawn(task);
         Ok(store)
     }
@@ -294,11 +298,11 @@ mod tests {
             clean_region_threshold: 1,
         };
 
-        let (store, task) = LazyStorage::<_, _, Store<_, _>>::with_task(config.into());
+        let (store, handle) = LazyStorage::<_, _, Store<_, _>>::with_handle(config.into());
 
         assert!(!store.insert(100, 100).await.unwrap());
 
-        tokio::spawn(task).await.unwrap().unwrap();
+        handle.await.unwrap().unwrap();
 
         assert!(store.insert(100, 100).await.unwrap());
         assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
@@ -329,11 +333,11 @@ mod tests {
             clean_region_threshold: 1,
         };
 
-        let (store, task) = LazyStorage::<_, _, Store<_, _>>::with_task(config.into());
+        let (store, handle) = LazyStorage::<_, _, Store<_, _>>::with_handle(config.into());
 
         assert!(store.lookup(&100).await.unwrap().is_none());
 
-        tokio::spawn(task).await.unwrap().unwrap();
+        handle.await.unwrap().unwrap();
 
         assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
     }
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 9e9d68a2..a957834e 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -21,6 +21,7 @@ use foyer_common::{
 
 use crate::{
     error::Result,
+    lazy::LazyStore,
     storage::{Storage, StorageWriter},
     store::Store,
 };
@@ -142,6 +143,10 @@ where
         Ok(Self { runtime, store })
     }
 
+    fn is_ready(&self) -> bool {
+        self.store.is_ready()
+    }
+
     async fn close(&self) -> Result<()> {
         let store = self.store.clone();
         self.runtime
@@ -183,3 +188,7 @@ where
 pub type RuntimeStore<K, V> = RuntimeStorage<K, V, Store<K, V>>;
 pub type RuntimeStoreWriter<K, V> = RuntimeStorageWriter<K, V, Store<K, V>>;
 pub type RuntimeStoreConfig<K, V> = RuntimeStorageConfig<K, V, Store<K, V>>;
+
+pub type RuntimeLazyStore<K, V> = RuntimeStorage<K, V, LazyStore<K, V>>;
+pub type RuntimeLazyStoreWriter<K, V> = RuntimeStorageWriter<K, V, LazyStore<K, V>>;
+pub type RuntimeLazyStoreConfig<K, V> = RuntimeStorageConfig<K, V, LazyStore<K, V>>;
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index eff65695..4121f26b 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -39,9 +39,7 @@ pub trait Storage: Send + Sync + Debug + Clone + 'static {
     #[must_use]
     fn open(config: Self::Config) -> impl Future<Output = Result<Self>> + Send;
 
-    fn is_ready(&self) -> bool {
-        true
-    }
+    fn is_ready(&self) -> bool;
 
     #[must_use]
     fn close(&self) -> impl Future<Output = Result<()>> + Send;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 57b78805..545486f5 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -253,6 +253,14 @@ where
         }
     }
 
+    fn is_ready(&self) -> bool {
+        match self {
+            Store::LruFsStore { store } => store.is_ready(),
+            Store::LfuFsStore { store } => store.is_ready(),
+            Store::FifoFsStore { store } => store.is_ready(),
+        }
+    }
+
     async fn close(&self) -> Result<()> {
         match self {
             Store::LruFsStore { store } => store.close().await,
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index c6b48e73..f693719e 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -21,7 +21,7 @@ use foyer_intrusive::eviction::fifo::FifoConfig;
 use foyer_storage::{
     device::fs::FsDeviceConfig,
     lazy::LazyStore,
-    runtime::{RuntimeConfig, RuntimeStorageConfig, RuntimeStore},
+    runtime::{RuntimeConfig, RuntimeLazyStore, RuntimeStorageConfig, RuntimeStore},
     storage::{Storage, StorageExt},
     store::{FifoFsStoreConfig, Store},
     test_utils::JudgeRecorder,
@@ -211,3 +211,40 @@ async fn test_runtime_store() {
 
     test_storage::<RuntimeStore<_, _>>(config, recorder).await;
 }
+
+#[tokio::test]
+async fn test_runtime_lazy_store() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let recorder = Arc::new(JudgeRecorder::default());
+    let config = RuntimeStorageConfig {
+        store: FifoFsStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: PathBuf::from(tempdir.path()),
+                capacity: 4 * MB,
+                file_capacity: 1 * MB,
+                align: 4 * KB,
+                io_size: 4 * KB,
+            },
+            allocator_bits: 0,
+            admissions: vec![recorder.clone()],
+            reinsertions: vec![recorder.clone()],
+            buffer_pool_size: 2 * MB,
+            flushers: 1,
+            flush_rate_limit: 0,
+            reclaimers: 1,
+            reclaim_rate_limit: 0,
+            allocation_timeout: Duration::from_millis(10),
+            clean_region_threshold: 1,
+            recover_concurrency: 2,
+        }
+        .into(),
+        runtime: RuntimeConfig {
+            worker_threads: None,
+            thread_name: None,
+        },
+    };
+
+    test_storage::<RuntimeLazyStore<_, _>>(config, recorder).await;
+}

From c92c6a0c1e3cfacf205cd118c3d9a5ffdf2d2d82 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 8 Oct 2023 22:45:33 -0500
Subject: [PATCH 127/261] refactor: move NoneStore into Store (#161)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/lazy.rs  |  78 ++-----------------------
 foyer-storage/src/store.rs | 114 +++++++++++++++++++++++++++++++++++++
 2 files changed, 119 insertions(+), 73 deletions(-)

diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 7d17737e..ff56f60c 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -12,84 +12,16 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    marker::PhantomData,
-    sync::{Arc, OnceLock},
-};
+use std::sync::{Arc, OnceLock};
 
 use crate::{
     error::Result,
     storage::{Storage, StorageWriter},
-    store::Store,
+    store::{NoneStore, NoneStoreWriter, Store},
 };
 use foyer_common::code::{Key, Value};
 use tokio::task::JoinHandle;
 
-#[derive(Debug)]
-pub struct NoneStoreWriter<K: Key, V: Value>(PhantomData<(K, V)>);
-
-impl<K: Key, V: Value> StorageWriter for NoneStoreWriter<K, V> {
-    type Key = K;
-    type Value = V;
-
-    fn judge(&mut self) -> bool {
-        false
-    }
-
-    async fn finish(self, _: Self::Value) -> Result<bool> {
-        Ok(false)
-    }
-}
-
-#[derive(Debug)]
-pub struct NoneStore<K: Key, V: Value>(PhantomData<(K, V)>);
-
-impl<K: Key, V: Value> Clone for NoneStore<K, V> {
-    fn clone(&self) -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<K: Key, V: Value> Storage for NoneStore<K, V> {
-    type Key = K;
-    type Value = V;
-    type Config = ();
-    type Writer = NoneStoreWriter<K, V>;
-
-    #[expect(clippy::let_unit_value)]
-    async fn open(_: Self::Config) -> Result<Self> {
-        Ok(NoneStore(PhantomData))
-    }
-
-    fn is_ready(&self) -> bool {
-        true
-    }
-
-    async fn close(&self) -> Result<()> {
-        Ok(())
-    }
-
-    fn writer(&self, _: Self::Key, _: usize) -> Self::Writer {
-        NoneStoreWriter(PhantomData)
-    }
-
-    fn exists(&self, _: &Self::Key) -> Result<bool> {
-        Ok(false)
-    }
-
-    async fn lookup(&self, _: &Self::Key) -> Result<Option<Self::Value>> {
-        Ok(None)
-    }
-
-    fn remove(&self, _: &Self::Key) -> Result<bool> {
-        Ok(false)
-    }
-
-    fn clear(&self) -> Result<()> {
-        Ok(())
-    }
-}
-
 #[derive(Debug)]
 pub enum LazyStorageWriter<K, V, S>
 where
@@ -145,7 +77,7 @@ where
     fn clone(&self) -> Self {
         Self {
             once: Arc::clone(&self.once),
-            none: NoneStore(PhantomData),
+            none: NoneStore::default(),
         }
     }
 }
@@ -176,7 +108,7 @@ where
 
         let res = Self {
             once,
-            none: NoneStore(PhantomData),
+            none: NoneStore::default(),
         };
 
         (res, handle)
@@ -217,7 +149,7 @@ where
                 writer: store.writer(key, weight),
             },
             None => LazyStorageWriter::None {
-                writer: NoneStoreWriter(PhantomData),
+                writer: NoneStoreWriter::default(),
             },
         }
     }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 545486f5..aeb6fa60 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -12,6 +12,8 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::marker::PhantomData;
+
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::eviction::{
     fifo::{Fifo, FifoLink},
@@ -54,6 +56,87 @@ pub type FifoFsStoreConfig<K, V> =
 pub type FifoFsStoreWriter<K, V> =
     GenericStoreWriter<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
+#[derive(Debug)]
+pub struct NoneStoreWriter<K: Key, V: Value>(PhantomData<(K, V)>);
+
+impl<K: Key, V: Value> Default for NoneStoreWriter<K, V> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<K: Key, V: Value> StorageWriter for NoneStoreWriter<K, V> {
+    type Key = K;
+    type Value = V;
+
+    fn judge(&mut self) -> bool {
+        false
+    }
+
+    async fn finish(self, _: Self::Value) -> Result<bool> {
+        Ok(false)
+    }
+}
+
+impl<K: Key, V: Value> ForceStorageWriter for NoneStoreWriter<K, V> {
+    fn set_force(&mut self) {}
+}
+
+#[derive(Debug)]
+pub struct NoneStore<K: Key, V: Value>(PhantomData<(K, V)>);
+
+impl<K: Key, V: Value> Default for NoneStore<K, V> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<K: Key, V: Value> Clone for NoneStore<K, V> {
+    fn clone(&self) -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<K: Key, V: Value> Storage for NoneStore<K, V> {
+    type Key = K;
+    type Value = V;
+    type Config = ();
+    type Writer = NoneStoreWriter<K, V>;
+
+    #[expect(clippy::let_unit_value)]
+    async fn open(_: Self::Config) -> Result<Self> {
+        Ok(NoneStore(PhantomData))
+    }
+
+    fn is_ready(&self) -> bool {
+        true
+    }
+
+    async fn close(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn writer(&self, _: Self::Key, _: usize) -> Self::Writer {
+        NoneStoreWriter(PhantomData)
+    }
+
+    fn exists(&self, _: &Self::Key) -> Result<bool> {
+        Ok(false)
+    }
+
+    async fn lookup(&self, _: &Self::Key) -> Result<Option<Self::Value>> {
+        Ok(None)
+    }
+
+    fn remove(&self, _: &Self::Key) -> Result<bool> {
+        Ok(false)
+    }
+
+    fn clear(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
 #[derive(Debug)]
 pub enum StoreConfig<K, V>
 where
@@ -63,6 +146,7 @@ where
     LruFsStoreConfig { config: LruFsStoreConfig<K, V> },
     LfuFsStoreConfig { config: LfuFsStoreConfig<K, V> },
     FifoFsStoreConfig { config: FifoFsStoreConfig<K, V> },
+    NoneStoreConfig,
 }
 
 impl<K, V> Clone for StoreConfig<K, V>
@@ -81,6 +165,7 @@ where
             Self::FifoFsStoreConfig { config } => Self::FifoFsStoreConfig {
                 config: config.clone(),
             },
+            Self::NoneStoreConfig => Self::NoneStoreConfig,
         }
     }
 }
@@ -124,6 +209,7 @@ where
     LruFsStorWriter { writer: LruFsStoreWriter<K, V> },
     LfuFsStorWriter { writer: LfuFsStoreWriter<K, V> },
     FifoFsStoreWriter { writer: FifoFsStoreWriter<K, V> },
+    NoneStoreWriter { writer: NoneStoreWriter<K, V> },
 }
 
 impl<K, V> From<LruFsStoreWriter<K, V>> for StoreWriter<K, V>
@@ -156,6 +242,16 @@ where
     }
 }
 
+impl<K, V> From<NoneStoreWriter<K, V>> for StoreWriter<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(writer: NoneStoreWriter<K, V>) -> Self {
+        StoreWriter::NoneStoreWriter { writer }
+    }
+}
+
 #[derive(Debug)]
 pub enum Store<K, V>
 where
@@ -165,6 +261,7 @@ where
     LruFsStore { store: LruFsStore<K, V> },
     LfuFsStore { store: LfuFsStore<K, V> },
     FifoFsStore { store: FifoFsStore<K, V> },
+    NoneStore { store: NoneStore<K, V> },
 }
 
 impl<K, V> Clone for Store<K, V>
@@ -183,6 +280,9 @@ where
             Self::FifoFsStore { store } => Self::FifoFsStore {
                 store: store.clone(),
             },
+            Self::NoneStore { store } => Self::NoneStore {
+                store: store.clone(),
+            },
         }
     }
 }
@@ -200,6 +300,7 @@ where
             StoreWriter::LruFsStorWriter { writer } => writer.judge(),
             StoreWriter::LfuFsStorWriter { writer } => writer.judge(),
             StoreWriter::FifoFsStoreWriter { writer } => writer.judge(),
+            StoreWriter::NoneStoreWriter { writer } => writer.judge(),
         }
     }
 
@@ -208,6 +309,7 @@ where
             StoreWriter::LruFsStorWriter { writer } => writer.finish(value).await,
             StoreWriter::LfuFsStorWriter { writer } => writer.finish(value).await,
             StoreWriter::FifoFsStoreWriter { writer } => writer.finish(value).await,
+            StoreWriter::NoneStoreWriter { writer } => writer.finish(value).await,
         }
     }
 }
@@ -222,6 +324,7 @@ where
             StoreWriter::LruFsStorWriter { writer } => writer.set_force(),
             StoreWriter::LfuFsStorWriter { writer } => writer.set_force(),
             StoreWriter::FifoFsStoreWriter { writer } => writer.set_force(),
+            StoreWriter::NoneStoreWriter { writer } => writer.set_force(),
         }
     }
 }
@@ -250,6 +353,10 @@ where
                 let store = FifoFsStore::open(config).await?;
                 Ok(Self::FifoFsStore { store })
             }
+            StoreConfig::NoneStoreConfig => {
+                let store = NoneStore::open(()).await?;
+                Ok(Self::NoneStore { store })
+            }
         }
     }
 
@@ -258,6 +365,7 @@ where
             Store::LruFsStore { store } => store.is_ready(),
             Store::LfuFsStore { store } => store.is_ready(),
             Store::FifoFsStore { store } => store.is_ready(),
+            Store::NoneStore { store } => store.is_ready(),
         }
     }
 
@@ -266,6 +374,7 @@ where
             Store::LruFsStore { store } => store.close().await,
             Store::LfuFsStore { store } => store.close().await,
             Store::FifoFsStore { store } => store.close().await,
+            Store::NoneStore { store } => store.close().await,
         }
     }
 
@@ -274,6 +383,7 @@ where
             Store::LruFsStore { store } => store.writer(key, weight).into(),
             Store::LfuFsStore { store } => store.writer(key, weight).into(),
             Store::FifoFsStore { store } => store.writer(key, weight).into(),
+            Store::NoneStore { store } => store.writer(key, weight).into(),
         }
     }
 
@@ -282,6 +392,7 @@ where
             Store::LruFsStore { store } => store.exists(key),
             Store::LfuFsStore { store } => store.exists(key),
             Store::FifoFsStore { store } => store.exists(key),
+            Store::NoneStore { store } => store.exists(key),
         }
     }
 
@@ -290,6 +401,7 @@ where
             Store::LruFsStore { store } => store.lookup(key).await,
             Store::LfuFsStore { store } => store.lookup(key).await,
             Store::FifoFsStore { store } => store.lookup(key).await,
+            Store::NoneStore { store } => store.lookup(key).await,
         }
     }
 
@@ -298,6 +410,7 @@ where
             Store::LruFsStore { store } => store.remove(key),
             Store::LfuFsStore { store } => store.remove(key),
             Store::FifoFsStore { store } => store.remove(key),
+            Store::NoneStore { store } => store.remove(key),
         }
     }
 
@@ -306,6 +419,7 @@ where
             Store::LruFsStore { store } => store.clear(),
             Store::LfuFsStore { store } => store.clear(),
             Store::FifoFsStore { store } => store.clear(),
+            Store::NoneStore { store } => store.clear(),
         }
     }
 }

From 168561bbae984280da06b55caa6b46b628ef1afb Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 9 Oct 2023 02:34:49 -0500
Subject: [PATCH 128/261] feat: introduce async storage ext (#162)

* feat: introduce async storage ext

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* rename fn

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/storage.rs | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 4121f26b..ff07599f 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -189,6 +189,21 @@ pub trait StorageExt: Storage {
 
 impl<S: Storage> StorageExt for S {}
 
+pub trait AsyncStorageExt: Storage {
+    #[tracing::instrument(skip(self, value))]
+    fn insert_async(&self, key: Self::Key, value: Self::Value) {
+        let weight = key.serialized_len() + value.serialized_len();
+        let store = self.clone();
+        tokio::spawn(async move {
+            if let Err(e) = store.writer(key, weight).finish(value).await {
+                tracing::warn!("async storage insert error: {}", e);
+            }
+        });
+    }
+}
+
+impl<S: Storage> AsyncStorageExt for S {}
+
 pub trait ForceStorageWriter: StorageWriter {
     fn set_force(&mut self);
 }

From 8933a3deb84de7595782eef6aedd68ccd4eb5c13 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 9 Oct 2023 02:41:37 -0500
Subject: [PATCH 129/261] feat: expose writer key & weight (#163)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/generic.rs |  8 +++++++
 foyer-storage/src/lazy.rs    | 16 ++++++++++++-
 foyer-storage/src/runtime.rs |  8 +++++++
 foyer-storage/src/storage.rs |  4 ++++
 foyer-storage/src/store.rs   | 46 +++++++++++++++++++++++++++++++-----
 5 files changed, 75 insertions(+), 7 deletions(-)

diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index d18149fe..d54730f8 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -992,6 +992,14 @@ where
     type Key = K;
     type Value = V;
 
+    fn key(&self) -> &Self::Key {
+        &self.key
+    }
+
+    fn weight(&self) -> usize {
+        self.weight
+    }
+
     fn judge(&mut self) -> bool {
         self.judge()
     }
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index ff56f60c..a6bb7190 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -42,6 +42,20 @@ where
     type Key = K;
     type Value = V;
 
+    fn key(&self) -> &Self::Key {
+        match self {
+            LazyStorageWriter::Store { writer } => writer.key(),
+            LazyStorageWriter::None { writer } => writer.key(),
+        }
+    }
+
+    fn weight(&self) -> usize {
+        match self {
+            LazyStorageWriter::Store { writer } => writer.weight(),
+            LazyStorageWriter::None { writer } => writer.weight(),
+        }
+    }
+
     fn judge(&mut self) -> bool {
         match self {
             LazyStorageWriter::Store { writer } => writer.judge(),
@@ -149,7 +163,7 @@ where
                 writer: store.writer(key, weight),
             },
             None => LazyStorageWriter::None {
-                writer: NoneStoreWriter::default(),
+                writer: NoneStoreWriter::new(key, weight),
             },
         }
     }
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index a957834e..5490585c 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -77,6 +77,14 @@ where
     type Key = K;
     type Value = V;
 
+    fn key(&self) -> &Self::Key {
+        self.writer.key()
+    }
+
+    fn weight(&self) -> usize {
+        self.writer.weight()
+    }
+
     fn judge(&mut self) -> bool {
         self.writer.judge()
     }
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index ff07599f..04e3b740 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -25,6 +25,10 @@ pub trait StorageWriter: Send + Sync + Debug {
     type Key: Key;
     type Value: Value;
 
+    fn key(&self) -> &Self::Key;
+
+    fn weight(&self) -> usize;
+
     fn judge(&mut self) -> bool;
 
     fn finish(self, value: Self::Value) -> impl Future<Output = Result<bool>> + Send;
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index aeb6fa60..e27b1e24 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -57,11 +57,19 @@ pub type FifoFsStoreWriter<K, V> =
     GenericStoreWriter<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
 #[derive(Debug)]
-pub struct NoneStoreWriter<K: Key, V: Value>(PhantomData<(K, V)>);
+pub struct NoneStoreWriter<K: Key, V: Value> {
+    key: K,
+    weight: usize,
+    _marker: PhantomData<V>,
+}
 
-impl<K: Key, V: Value> Default for NoneStoreWriter<K, V> {
-    fn default() -> Self {
-        Self(PhantomData)
+impl<K: Key, V: Value> NoneStoreWriter<K, V> {
+    pub fn new(key: K, weight: usize) -> Self {
+        Self {
+            key,
+            weight,
+            _marker: PhantomData,
+        }
     }
 }
 
@@ -69,6 +77,14 @@ impl<K: Key, V: Value> StorageWriter for NoneStoreWriter<K, V> {
     type Key = K;
     type Value = V;
 
+    fn key(&self) -> &Self::Key {
+        &self.key
+    }
+
+    fn weight(&self) -> usize {
+        self.weight
+    }
+
     fn judge(&mut self) -> bool {
         false
     }
@@ -116,8 +132,8 @@ impl<K: Key, V: Value> Storage for NoneStore<K, V> {
         Ok(())
     }
 
-    fn writer(&self, _: Self::Key, _: usize) -> Self::Writer {
-        NoneStoreWriter(PhantomData)
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
+        NoneStoreWriter::new(key, weight)
     }
 
     fn exists(&self, _: &Self::Key) -> Result<bool> {
@@ -295,6 +311,24 @@ where
     type Key = K;
     type Value = V;
 
+    fn key(&self) -> &Self::Key {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.key(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.key(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.key(),
+            StoreWriter::NoneStoreWriter { writer } => writer.key(),
+        }
+    }
+
+    fn weight(&self) -> usize {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.weight(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.weight(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.weight(),
+            StoreWriter::NoneStoreWriter { writer } => writer.weight(),
+        }
+    }
+
     fn judge(&mut self) -> bool {
         match self {
             StoreWriter::LruFsStorWriter { writer } => writer.judge(),

From feb1306033b88a595127afa0ab1e3a9dce80b907 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 10 Oct 2023 02:35:21 -0500
Subject: [PATCH 130/261] feat: support madsim (#164)

* feat: support madsim

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix hakari

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix Debug

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* complete replace tokio with madsim

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml       | 10 +++++-----
 .github/workflows/main.yml          | 10 +++++-----
 .github/workflows/pull-request.yml  | 10 +++++-----
 Cargo.toml                          | 10 ++++++++++
 Makefile                            |  7 ++++++-
 foyer-common/Cargo.toml             |  9 +--------
 foyer-common/src/runtime.rs         | 12 +++++++++++-
 foyer-memory/Cargo.toml             |  8 --------
 foyer-storage/Cargo.toml            |  9 +--------
 foyer-storage/src/device/mod.rs     |  3 +++
 foyer-storage/tests/storage_test.rs |  3 +--
 foyer-workspace-hack/Cargo.toml     |  4 ++--
 12 files changed, 50 insertions(+), 45 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index c55c64c4..4a07ad1f 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -5,7 +5,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230910
+  CACHE_KEY_SUFFIX: 20231010
 
 jobs:
   misc-check:
@@ -58,6 +58,10 @@ jobs:
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
+      - name: Run hakari check
+        run: |
+          cargo hakari generate --diff
+          cargo hakari manage-deps --dry-run
       - name: Run rust format check
         run: |
           cargo fmt --all -- --check
@@ -66,10 +70,6 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
-      - name: Run hakari check
-        run: |
-          cargo hakari generate --diff
-          cargo hakari manage-deps --dry-run
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 61fa082f..fdd4a5c8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230910
+  CACHE_KEY_SUFFIX: 20231010
 jobs:
   misc-check:
     name: misc check
@@ -65,6 +65,10 @@ jobs:
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
+      - name: Run hakari check
+        run: |
+          cargo hakari generate --diff
+          cargo hakari manage-deps --dry-run
       - name: Run rust format check
         run: |
           cargo fmt --all -- --check
@@ -73,10 +77,6 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
-      - name: Run hakari check
-        run: |
-          cargo hakari generate --diff
-          cargo hakari manage-deps --dry-run
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index e4f95dfa..fa8fcf5d 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -12,7 +12,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20230910
+  CACHE_KEY_SUFFIX: 20231010
 jobs:
   misc-check:
     name: misc check
@@ -64,6 +64,10 @@ jobs:
       - name: Run rust cargo-sort check
         run: |
           cargo sort -w -c
+      - name: Run hakari check
+        run: |
+          cargo hakari generate --diff
+          cargo hakari manage-deps --dry-run
       - name: Run rust format check
         run: |
           cargo fmt --all -- --check
@@ -72,10 +76,6 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets -- -D warnings
-      - name: Run hakari check
-        run: |
-          cargo hakari generate --diff
-          cargo hakari manage-deps --dry-run
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
       - if: steps.cache.outputs.cache-hit != 'true'
diff --git a/Cargo.toml b/Cargo.toml
index 54508932..6af6dc06 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,16 @@ members = [
     "foyer-workspace-hack",
 ]
 
+[workspace.dependencies]
+tokio = { package = "madsim-tokio", version = "0.2", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+
 [patch.crates-io]
 # cmsketch = { path = "../cmsketch-rs" }
 
diff --git a/Makefile b/Makefile
index ad07a9a4..ddc7728a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: deps check test test-ignored test-all all monitor clear
+.PHONY: deps check test test-ignored test-all all monitor clear madsim
 
 deps:
 	./scripts/install-deps.sh
@@ -25,6 +25,11 @@ test-ignored:
 
 test-all: test test-ignored
 
+madsim:
+	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" cargo clippy --all-targets
+	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" RUST_BACKTRACE=1 cargo nextest run --all
+	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" RUST_BACKTRACE=1 cargo test --doc
+
 all: check test-all
 
 monitor:
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index a58a1e36..2fd9bf0b 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -16,14 +16,7 @@ foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 rand = "0.8.5"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
+tokio = { workspace = true }
 tracing = "0.1"
 
 [dev-dependencies]
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index c299f60a..8b002b84 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -13,6 +13,7 @@
 //  limitations under the License.
 
 use std::{
+    fmt::Debug,
     mem::ManuallyDrop,
     ops::{Deref, DerefMut},
 };
@@ -22,13 +23,22 @@ use tokio::runtime::Runtime;
 /// A wrapper around [`Runtime`] that shuts down the runtime in the background when dropped.
 ///
 /// This is necessary because directly dropping a nested runtime is not allowed in a parent runtime.
-#[derive(Debug)]
 pub struct BackgroundShutdownRuntime(ManuallyDrop<Runtime>);
 
+impl Debug for BackgroundShutdownRuntime {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("BackgroundShutdownRuntime").finish()
+    }
+}
+
 impl Drop for BackgroundShutdownRuntime {
     fn drop(&mut self) {
         // Safety: The runtime is only dropped once here.
         let runtime = unsafe { ManuallyDrop::take(&mut self.0) };
+
+        #[cfg(madsim)]
+        drop(runtime);
+        #[cfg(not(madsim))]
         runtime.shutdown_background();
     }
 }
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 298ef90d..2dd2de38 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -21,14 +21,6 @@ memoffset = "0.9"
 parking_lot = "0.12"
 paste = "1.0"
 rand = "0.8.5"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
 tracing = "0.1"
 twox-hash = "1"
 
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index f31dd1f9..e98544a2 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -30,14 +30,7 @@ paste = "1.0"
 prometheus = "0.13"
 rand = "0.8.5"
 thiserror = "1"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
+tokio = { workspace = true }
 tracing = "0.1"
 twox-hash = "1"
 
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index a1fb1429..dc0ef19e 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -78,10 +78,13 @@ where
     F: FnOnce() -> DeviceResult<T> + Send + 'static,
     T: Send + 'static,
 {
+    #[cfg(not(madsim))]
     match tokio::task::spawn_blocking(f).await {
         Ok(res) => res,
         Err(e) => Err(format!("background task failed: {:?}", e,).into()),
     }
+    #[cfg(madsim)]
+    f()
 }
 
 #[cfg(test)]
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index f693719e..3ab87242 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -15,8 +15,6 @@
 #![feature(lint_reasons)]
 #![expect(clippy::identity_op)]
 
-use std::{path::PathBuf, sync::Arc, time::Duration};
-
 use foyer_intrusive::eviction::fifo::FifoConfig;
 use foyer_storage::{
     device::fs::FsDeviceConfig,
@@ -26,6 +24,7 @@ use foyer_storage::{
     store::{FifoFsStoreConfig, Store},
     test_utils::JudgeRecorder,
 };
+use std::{path::PathBuf, sync::Arc, time::Duration};
 
 const KB: usize = 1024;
 const MB: usize = 1024 * 1024;
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index b35b807a..bf3e6df1 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -27,8 +27,8 @@ parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"]
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
 regex = { version = "1" }
-regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
-regex-syntax = { version = "0.7" }
+regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
+regex-syntax = { version = "0.8" }
 tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 tracing = { version = "0.1" }
 tracing-core = { version = "0.1" }

From 89456d55de3880c3e3b062976e5d15f920eb53b7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 10 Oct 2023 03:16:21 -0500
Subject: [PATCH 131/261] chore: enable deterministic test on CI (#165)

* chore: enable deterministic test on CI

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update ci cache key

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* separate ci cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      | 47 +++++++++++++++++++++++++++---
 .github/workflows/main.yml         | 47 +++++++++++++++++++++++++++---
 .github/workflows/pull-request.yml | 47 +++++++++++++++++++++++++++---
 3 files changed, 129 insertions(+), 12 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 4a07ad1f..a9debcce 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -5,7 +5,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231010
+  CACHE_KEY_SUFFIX: 20231010v3
 
 jobs:
   misc-check:
@@ -46,7 +46,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-test
       - name: Install cargo-binstall
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -101,7 +101,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -131,7 +131,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
@@ -141,3 +141,42 @@ jobs:
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+  deterministic-test:
+    name: run deterministic test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust clippy check (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo clippy --all-targets
+      - name: Run nexmark test (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo nextest run --all
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index fdd4a5c8..5aad8b4f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231010
+  CACHE_KEY_SUFFIX: 20231010v3
 jobs:
   misc-check:
     name: misc check
@@ -53,7 +53,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-test
       - name: Install cargo-binstall
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -108,7 +108,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -138,7 +138,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
@@ -148,6 +148,45 @@ jobs:
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+  deterministic-test:
+    name: run deterministic test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust clippy check (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo clippy --all-targets
+      - name: Run nexmark test (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo nextest run --all
 
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index fa8fcf5d..d5e50f1d 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -12,7 +12,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-09-09
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231010
+  CACHE_KEY_SUFFIX: 20231010v3
 jobs:
   misc-check:
     name: misc check
@@ -52,7 +52,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-test
       - name: Install cargo-binstall
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -107,7 +107,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -137,7 +137,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
@@ -147,6 +147,45 @@ jobs:
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+  deterministic-test:
+    name: run deterministic test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust clippy check (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo clippy --all-targets
+      - name: Run nexmark test (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo nextest run --all
 concurrency:
   group: environment-${{ github.ref }}
   cancel-in-progress: true

From 91ef64bb76f53182b1bd8b5feeeca553036e5bd7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 10 Oct 2023 03:45:35 -0500
Subject: [PATCH 132/261] chore: rename bench arg (#166)

* chore: rename bench arg

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update comment

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index edb3d5ca..4a444d6d 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -130,25 +130,25 @@ pub struct Args {
     #[arg(long, default_value_t = 16)]
     recover_concurrency: usize,
 
-    /// enable rated random admission policy if `rated_random` > 0
+    /// enable rated random admission policy if `random_insert_rate_limit` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
-    rated_random_admission: usize,
+    random_insert_rate_limit: usize,
 
-    /// enable rated random reinsertion policy if `rated_random` > 0
+    /// enable rated random reinsertion policy if `random_reinsert_rate_limit` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
-    rated_random_reinsertion: usize,
+    random_reinsert_rate_limit: usize,
 
-    /// enable rated ticket admission policy if `rated_random` > 0
+    /// enable rated ticket admission policy if `ticket_insert_rate_limit` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
-    rated_ticket_admission: usize,
+    ticket_insert_rate_limit: usize,
 
-    /// enable rated ticket reinsetion policy if `rated_random` > 0
+    /// enable rated ticket reinsetion policy if `ticket_reinsert_rate_limit` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
-    rated_ticket_reinsertion: usize,
+    ticket_reinsert_rate_limit: usize,
 
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
@@ -311,26 +311,26 @@ async fn main() {
 
     let mut admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
     let mut reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
-    if args.rated_random_admission > 0 {
+    if args.random_insert_rate_limit > 0 {
         let rr = RatedRandomAdmissionPolicy::new(
-            args.rated_random_admission * 1024 * 1024,
+            args.random_insert_rate_limit * 1024 * 1024,
             Duration::from_millis(100),
         );
         admissions.push(Arc::new(rr));
     }
-    if args.rated_random_reinsertion > 0 {
+    if args.random_reinsert_rate_limit > 0 {
         let rr = RatedRandomReinsertionPolicy::new(
-            args.rated_random_reinsertion * 1024 * 1024,
+            args.random_reinsert_rate_limit * 1024 * 1024,
             Duration::from_millis(100),
         );
         reinsertions.push(Arc::new(rr));
     }
-    if args.rated_ticket_admission > 0 {
-        let rt = RatedTicketAdmissionPolicy::new(args.rated_ticket_admission * 1024 * 1024);
+    if args.ticket_insert_rate_limit > 0 {
+        let rt = RatedTicketAdmissionPolicy::new(args.ticket_insert_rate_limit * 1024 * 1024);
         admissions.push(Arc::new(rt));
     }
-    if args.rated_ticket_reinsertion > 0 {
-        let rt = RatedTicketReinsertionPolicy::new(args.rated_ticket_reinsertion * 1024 * 1024);
+    if args.ticket_reinsert_rate_limit > 0 {
+        let rt = RatedTicketReinsertionPolicy::new(args.ticket_reinsert_rate_limit * 1024 * 1024);
         reinsertions.push(Arc::new(rt));
     }
 

From 9100e090a76f01747335cdd8728b40c10b652993 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 10 Oct 2023 04:36:50 -0500
Subject: [PATCH 133/261] feat: bench tool support w/wo separate runtime (#167)

* feat: bench tool support w/wo separate runtime

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refine comment

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/Cargo.toml  |   1 +
 foyer-storage-bench/src/main.rs | 248 ++++++++++++++++++++++++++++++--
 foyer-storage/src/generic.rs    |   1 +
 3 files changed, 235 insertions(+), 15 deletions(-)

diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 195e6fcd..30c1a747 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -15,6 +15,7 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.1", optional = true }
+foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-storage = { path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 4a444d6d..03276da6 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -14,6 +14,7 @@
 
 #![feature(let_chains)]
 #![feature(lint_reasons)]
+#![feature(async_fn_in_trait)]
 
 mod analyze;
 mod export;
@@ -34,6 +35,7 @@ use std::{
 use analyze::{analyze, monitor, Metrics};
 use clap::Parser;
 
+use foyer_common::code::{Key, Value};
 use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
     admission::{
@@ -41,13 +43,14 @@ use foyer_storage::{
         AdmissionPolicy,
     },
     device::fs::FsDeviceConfig,
-    generic::GenericStoreConfig,
+    error::Result,
     reinsertion::{
         rated_random::RatedRandomReinsertionPolicy, rated_ticket::RatedTicketReinsertionPolicy,
         ReinsertionPolicy,
     },
-    storage::{Storage, StorageExt},
-    store::LfuFsStore,
+    runtime::{RuntimeConfig, RuntimeStore, RuntimeStoreConfig, RuntimeStoreWriter},
+    storage::{Storage, StorageExt, StorageWriter},
+    store::{LfuFsStoreConfig, Store, StoreConfig, StoreWriter},
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -145,7 +148,7 @@ pub struct Args {
     #[arg(long, default_value_t = 0)]
     ticket_insert_rate_limit: usize,
 
-    /// enable rated ticket reinsetion policy if `ticket_reinsert_rate_limit` > 0
+    /// enable rated ticket reinsetion policy if `ticket_reinsert_rate_limitgit a` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     ticket_reinsert_rate_limit: usize,
@@ -162,23 +165,217 @@ pub struct Args {
     #[arg(long, default_value_t = 10)]
     allocation_timeout: usize,
 
-    /// `0` means equal to reclaimer count.
+    /// `0` means equal to reclaimer count
     #[arg(long, default_value_t = 0)]
     clean_region_threshold: usize,
 
-    /// The count of allocators is `2 ^ allocator bits`.
+    /// the count of allocators is `2 ^ allocator bits`
     ///
     /// Note: The count of allocators should be greater than buffer count.
     ///       (buffer count = buffer pool size / device region size)
     #[arg(long, default_value_t = 0)]
     allocator_bits: usize,
 
-    /// Weigher to enable metrics exporter.
+    /// weigher to enable metrics exporter
     #[arg(long, default_value_t = false)]
     metrics: bool,
+
+    /// use separate runtime
+    #[arg(long, default_value_t = false)]
+    runtime: bool,
+}
+
+#[derive(Debug)]
+pub enum BenchStoreConfig<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    StoreConfig { config: StoreConfig<K, V> },
+    RuntimeStoreConfig { config: RuntimeStoreConfig<K, V> },
+}
+
+impl<K, V> Clone for BenchStoreConfig<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn clone(&self) -> Self {
+        match self {
+            Self::StoreConfig { config } => Self::StoreConfig {
+                config: config.clone(),
+            },
+            Self::RuntimeStoreConfig { config } => Self::RuntimeStoreConfig {
+                config: config.clone(),
+            },
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum BenchStoreWriter<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    StoreWriter { writer: StoreWriter<K, V> },
+    RuntimeStoreWriter { writer: RuntimeStoreWriter<K, V> },
+}
+
+impl<K, V> From<StoreWriter<K, V>> for BenchStoreWriter<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(writer: StoreWriter<K, V>) -> Self {
+        Self::StoreWriter { writer }
+    }
+}
+
+impl<K, V> From<RuntimeStoreWriter<K, V>> for BenchStoreWriter<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn from(writer: RuntimeStoreWriter<K, V>) -> Self {
+        Self::RuntimeStoreWriter { writer }
+    }
+}
+
+impl<K, V> StorageWriter for BenchStoreWriter<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+
+    fn key(&self) -> &Self::Key {
+        match self {
+            BenchStoreWriter::StoreWriter { writer } => writer.key(),
+            BenchStoreWriter::RuntimeStoreWriter { writer } => writer.key(),
+        }
+    }
+
+    fn weight(&self) -> usize {
+        match self {
+            BenchStoreWriter::StoreWriter { writer } => writer.weight(),
+            BenchStoreWriter::RuntimeStoreWriter { writer } => writer.weight(),
+        }
+    }
+
+    fn judge(&mut self) -> bool {
+        match self {
+            BenchStoreWriter::StoreWriter { writer } => writer.judge(),
+            BenchStoreWriter::RuntimeStoreWriter { writer } => writer.judge(),
+        }
+    }
+
+    async fn finish(self, value: Self::Value) -> Result<bool> {
+        match self {
+            BenchStoreWriter::StoreWriter { writer } => writer.finish(value).await,
+            BenchStoreWriter::RuntimeStoreWriter { writer } => writer.finish(value).await,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum BenchStore<K = u64, V = Vec<u8>>
+where
+    K: Key,
+    V: Value,
+{
+    Store { store: Store<K, V> },
+    RuntimeStore { store: RuntimeStore<K, V> },
 }
 
-type TStore = LfuFsStore<u64, Vec<u8>>;
+impl<K, V> Clone for BenchStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn clone(&self) -> Self {
+        match self {
+            Self::Store { store } => Self::Store {
+                store: store.clone(),
+            },
+            Self::RuntimeStore { store } => Self::RuntimeStore {
+                store: store.clone(),
+            },
+        }
+    }
+}
+
+impl<K, V> Storage for BenchStore<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Config = BenchStoreConfig<K, V>;
+    type Writer = BenchStoreWriter<K, V>;
+
+    async fn open(config: Self::Config) -> Result<Self> {
+        match config {
+            BenchStoreConfig::StoreConfig { config } => {
+                Store::open(config).await.map(|store| Self::Store { store })
+            }
+            BenchStoreConfig::RuntimeStoreConfig { config } => RuntimeStore::open(config)
+                .await
+                .map(|store| Self::RuntimeStore { store }),
+        }
+    }
+
+    fn is_ready(&self) -> bool {
+        match self {
+            BenchStore::Store { store } => store.is_ready(),
+            BenchStore::RuntimeStore { store } => store.is_ready(),
+        }
+    }
+
+    async fn close(&self) -> Result<()> {
+        match self {
+            BenchStore::Store { store } => store.close().await,
+            BenchStore::RuntimeStore { store } => store.close().await,
+        }
+    }
+
+    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
+        match self {
+            BenchStore::Store { store } => store.writer(key, weight).into(),
+            BenchStore::RuntimeStore { store } => store.writer(key, weight).into(),
+        }
+    }
+
+    fn exists(&self, key: &Self::Key) -> Result<bool> {
+        match self {
+            BenchStore::Store { store } => store.exists(key),
+            BenchStore::RuntimeStore { store } => store.exists(key),
+        }
+    }
+
+    async fn lookup(&self, key: &Self::Key) -> Result<Option<Self::Value>> {
+        match self {
+            BenchStore::Store { store } => store.lookup(key).await,
+            BenchStore::RuntimeStore { store } => store.lookup(key).await,
+        }
+    }
+
+    fn remove(&self, key: &Self::Key) -> Result<bool> {
+        match self {
+            BenchStore::Store { store } => store.remove(key),
+            BenchStore::RuntimeStore { store } => store.remove(key),
+        }
+    }
+
+    fn clear(&self) -> Result<()> {
+        match self {
+            BenchStore::Store { store } => store.clear(),
+            BenchStore::RuntimeStore { store } => store.clear(),
+        }
+    }
+}
 
 fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
@@ -243,7 +440,7 @@ fn init_logger() {
 
 #[tokio::main]
 async fn main() {
-    is_send_sync_static::<TStore>();
+    is_send_sync_static::<BenchStore>();
 
     init_logger();
 
@@ -340,7 +537,7 @@ async fn main() {
         args.clean_region_threshold
     };
 
-    let config = GenericStoreConfig {
+    let config = LfuFsStoreConfig {
         name: "".to_string(),
         eviction_config,
         device_config,
@@ -357,9 +554,25 @@ async fn main() {
         clean_region_threshold,
     };
 
-    println!("{:#?}", config);
+    let config = if args.runtime {
+        BenchStoreConfig::RuntimeStoreConfig {
+            config: RuntimeStoreConfig {
+                store: config.into(),
+                runtime: RuntimeConfig {
+                    worker_threads: None,
+                    thread_name: Some("foyer".to_string()),
+                },
+            },
+        }
+    } else {
+        BenchStoreConfig::StoreConfig {
+            config: config.into(),
+        }
+    };
+
+    println!("{config:#?}");
 
-    let store = TStore::open(config).await.unwrap();
+    let store = BenchStore::open(config).await.unwrap();
 
     let (stop_tx, _) = broadcast::channel(4096);
 
@@ -408,7 +621,12 @@ async fn main() {
     println!("\nTotal:\n{}", analysis);
 }
 
-async fn bench(args: Args, store: TStore, metrics: Metrics, stop_tx: broadcast::Sender<()>) {
+async fn bench(
+    args: Args,
+    store: impl Storage<Key = u64, Value = Vec<u8>>,
+    metrics: Metrics,
+    stop_tx: broadcast::Sender<()>,
+) {
     let w_rate = if args.w_rate == 0.0 {
         None
     } else {
@@ -457,7 +675,7 @@ async fn write(
     entry_size_range: Range<usize>,
     rate: Option<f64>,
     index: Arc<AtomicU64>,
-    store: TStore,
+    store: impl Storage<Key = u64, Value = Vec<u8>>,
     time: u64,
     metrics: Metrics,
     mut stop: broadcast::Receiver<()>,
@@ -502,7 +720,7 @@ async fn write(
 async fn read(
     rate: Option<f64>,
     index: Arc<AtomicU64>,
-    store: TStore,
+    store: impl Storage<Key = u64, Value = Vec<u8>>,
     time: u64,
     metrics: Metrics,
     mut stop: broadcast::Receiver<()>,
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index d54730f8..e5429256 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -539,6 +539,7 @@ where
         writer.is_judged = true;
     }
 
+    #[tracing::instrument(skip(self, value))]
     async fn apply_writer(
         &self,
         mut writer: GenericStoreWriter<K, V, D, EP, EL>,

From 1c1416d5b1c8afdc2e07e089733c07f3f2526cab Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 10 Oct 2023 10:39:54 -0500
Subject: [PATCH 134/261] chore: remove unused code (#168)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region.rs | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 1ba6d64b..ead41833 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{collections::HashMap, fmt::Debug, ops::RangeBounds, task::Waker};
+use std::{fmt::Debug, ops::RangeBounds};
 
 use bytes::{Buf, BufMut};
 use foyer_common::erwlock::{ErwLock, ErwLockInner};
@@ -79,8 +79,6 @@ where
     writers: usize,
     buffered_readers: usize,
     physical_readers: usize,
-
-    wakers: HashMap<usize, Waker>,
 }
 
 #[derive(Debug, Clone)]
@@ -139,8 +137,6 @@ where
             writers: 0,
             buffered_readers: 0,
             physical_readers: 0,
-
-            wakers: HashMap::default(),
         };
         Self {
             id,
@@ -156,7 +152,6 @@ where
             let f = move || {
                 let mut guard = inner.write();
                 guard.writers -= 1;
-                guard.wake_all();
             };
             Box::new(f)
         };
@@ -244,7 +239,6 @@ where
                     let f = move || {
                         let mut guard = inner.write();
                         guard.buffered_readers -= 1;
-                        guard.wake_all();
                     };
                     Box::new(f)
                 };
@@ -281,7 +275,6 @@ where
             {
                 let mut inner = self.inner.write();
                 inner.physical_readers -= 1;
-                inner.wake_all();
                 return Ok(None);
             }
             offset += len;
@@ -292,7 +285,6 @@ where
             let f = move || {
                 let mut guard = inner.write();
                 guard.physical_readers -= 1;
-                guard.wake_all();
             };
             Box::new(f)
         };
@@ -395,12 +387,6 @@ where
     pub fn physical_readers(&self) -> usize {
         self.physical_readers
     }
-
-    fn wake_all(&self) {
-        for waker in self.wakers.values() {
-            waker.wake_by_ref();
-        }
-    }
 }
 
 // read & write slice

From b98edb33e749909d6315a08c0e99580ebe10e3cd Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 10 Oct 2023 11:09:56 -0500
Subject: [PATCH 135/261] chore: clean code (#169)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region.rs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index ead41833..a28739ae 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -12,11 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, ops::RangeBounds};
-
 use bytes::{Buf, BufMut};
 use foyer_common::erwlock::{ErwLock, ErwLockInner};
 use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock};
+use std::{fmt::Debug, ops::RangeBounds};
 use tracing::instrument;
 
 use crate::{
@@ -289,7 +288,7 @@ where
             Box::new(f)
         };
         Ok(Some(ReadSlice::Owned {
-            buf: Some(buf),
+            buf,
             cleanup: Some(cleanup),
         }))
     }
@@ -464,7 +463,7 @@ where
         cleanup: Option<Box<dyn CleanupFn>>,
     },
     Owned {
-        buf: Option<Vec<u8, A>>,
+        buf: Vec<u8, A>,
         cleanup: Option<Box<dyn CleanupFn>>,
     },
 }
@@ -497,7 +496,7 @@ where
     pub fn len(&self) -> usize {
         match self {
             Self::Slice { slice, .. } => slice.len(),
-            Self::Owned { buf, .. } => buf.as_ref().unwrap().len(),
+            Self::Owned { buf, .. } => buf.len(),
         }
     }
 
@@ -513,7 +512,7 @@ where
     fn as_ref(&self) -> &[u8] {
         match self {
             Self::Slice { slice, .. } => slice.as_ref(),
-            Self::Owned { buf, .. } => buf.as_ref().unwrap(),
+            Self::Owned { buf, .. } => buf.as_ref(),
         }
     }
 }

From 75f3363a7707f0d138163347a926548907405574 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 10 Oct 2023 22:34:50 -0500
Subject: [PATCH 136/261] feat: merge concurrent physical read (#170)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region.rs | 102 ++++++++++++++++++++++++++++++------
 1 file changed, 87 insertions(+), 15 deletions(-)

diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index a28739ae..1a22d2e5 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -14,8 +14,14 @@
 
 use bytes::{Buf, BufMut};
 use foyer_common::erwlock::{ErwLock, ErwLockInner};
-use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock};
-use std::{fmt::Debug, ops::RangeBounds};
+use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock, RwLockWriteGuard};
+use std::{
+    collections::btree_map::{BTreeMap, Entry},
+    fmt::Debug,
+    ops::RangeBounds,
+    sync::Arc,
+};
+use tokio::sync::oneshot;
 use tracing::instrument;
 
 use crate::{
@@ -78,6 +84,9 @@ where
     writers: usize,
     buffered_readers: usize,
     physical_readers: usize,
+
+    #[expect(clippy::type_complexity)]
+    waits: BTreeMap<(usize, usize), Vec<oneshot::Sender<Result<ReadSlice<A>>>>>,
 }
 
 #[derive(Debug, Clone)]
@@ -136,6 +145,8 @@ where
             writers: 0,
             buffered_readers: 0,
             physical_readers: 0,
+
+            waits: BTreeMap::new(),
         };
         Self {
             id,
@@ -219,8 +230,10 @@ where
             std::ops::Bound::Unbounded => self.device.region_size(),
         };
 
+        // case 1: read from dirty buffer
+
         // restrict guard lifetime
-        {
+        let rx = {
             let mut inner = self.inner.write();
 
             if version != 0 && version != inner.version {
@@ -248,11 +261,31 @@ where
                 }));
             }
 
-            // if buffer detached, physical read
+            // case 2: join wait map if exists
+            let rx = match inner.waits.entry((start, end)) {
+                Entry::Vacant(v) => {
+                    v.insert(vec![]);
+                    None
+                }
+                Entry::Occupied(mut o) => {
+                    let (tx, rx) = oneshot::channel();
+                    o.get_mut().push(tx);
+                    Some(rx)
+                }
+            };
+
             inner.physical_readers += 1;
             drop(inner);
+
+            rx
+        };
+
+        // case 3: wait for result
+        if let Some(rx) = rx {
+            return rx.await.map_err(anyhow::Error::from)?.map(Some);
         }
 
+        // case 4: read from device
         let region = self.id;
         let mut buf = self.device.io_buffer(end - start, end - start);
 
@@ -266,18 +299,28 @@ where
                 start + offset + len
             );
             let s = unsafe { SliceMut::new(&mut buf[offset..offset + len]) };
-            if self
+            let read = match self
                 .device
                 .read(s, region, (start + offset) as u64, len)
-                .await?
-                != len
+                .await
             {
+                Ok(bytes) => bytes,
+                Err(e) => {
+                    let mut inner = self.inner.write();
+                    self.cleanup(&mut inner, start, end)?;
+                    inner.physical_readers -= 1;
+                    return Err(e.into());
+                }
+            };
+            if read != len {
                 let mut inner = self.inner.write();
+                self.cleanup(&mut inner, start, end)?;
                 inner.physical_readers -= 1;
                 return Ok(None);
             }
             offset += len;
         }
+        let buf = Arc::new(buf);
 
         let cleanup = {
             let inner = self.inner.clone();
@@ -287,7 +330,19 @@ where
             };
             Box::new(f)
         };
-        Ok(Some(ReadSlice::Owned {
+
+        if let Some(txs) = self.inner.write().waits.remove(&(start, end)) {
+            // TODO: handle error !!!!!!!!!!!
+            for tx in txs {
+                tx.send(Ok(ReadSlice::Shared {
+                    buf: buf.clone(),
+                    cleanup: Some(cleanup.clone()),
+                }))
+                .map_err(|_| anyhow::anyhow!("fail to send load result"))?;
+            }
+        }
+
+        Ok(Some(ReadSlice::Shared {
             buf,
             cleanup: Some(cleanup),
         }))
@@ -353,6 +408,23 @@ where
         inner.version += 1;
         res
     }
+
+    /// Cleanup waits.
+    fn cleanup(
+        &self,
+        guard: &mut RwLockWriteGuard<'_, RegionInner<D::IoBufferAllocator>>,
+        start: usize,
+        end: usize,
+    ) -> Result<()> {
+        if let Some(txs) = guard.waits.remove(&(start, end)) {
+            guard.writers -= txs.len();
+            for tx in txs {
+                tx.send(Err(anyhow::anyhow!("cancelled by previous error").into()))
+                    .map_err(|_| anyhow::anyhow!("fail to cleanup waits"))?;
+            }
+        }
+        Ok(())
+    }
 }
 
 impl<A> RegionInner<A>
@@ -462,8 +534,8 @@ where
         allocator: Option<A>,
         cleanup: Option<Box<dyn CleanupFn>>,
     },
-    Owned {
-        buf: Vec<u8, A>,
+    Shared {
+        buf: Arc<Vec<u8, A>>,
         cleanup: Option<Box<dyn CleanupFn>>,
     },
 }
@@ -481,8 +553,8 @@ where
                 .field("slice", slice)
                 .field("allocator", allocator)
                 .finish(),
-            Self::Owned { buf, .. } => f
-                .debug_struct("ReadSlice::Owned")
+            Self::Shared { buf, .. } => f
+                .debug_struct("ReadSlice::Shared")
                 .field("buf", buf)
                 .finish(),
         }
@@ -496,7 +568,7 @@ where
     pub fn len(&self) -> usize {
         match self {
             Self::Slice { slice, .. } => slice.len(),
-            Self::Owned { buf, .. } => buf.len(),
+            Self::Shared { buf, .. } => buf.len(),
         }
     }
 
@@ -512,7 +584,7 @@ where
     fn as_ref(&self) -> &[u8] {
         match self {
             Self::Slice { slice, .. } => slice.as_ref(),
-            Self::Owned { buf, .. } => buf.as_ref(),
+            Self::Shared { buf, .. } => buf.as_ref(),
         }
     }
 }
@@ -524,7 +596,7 @@ where
     fn drop(&mut self) {
         if let Some(f) = match self {
             ReadSlice::Slice { cleanup, .. } => cleanup.take(),
-            ReadSlice::Owned { cleanup, .. } => cleanup.take(),
+            ReadSlice::Shared { cleanup, .. } => cleanup.take(),
         } {
             f();
         }

From f097abed38e4363a6f2310760e304577c13cd68a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 11 Oct 2023 00:55:44 -0500
Subject: [PATCH 137/261] chore: fix acquire clean buffer metrics (#171)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region_manager.rs | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 33fc6007..7f09bf4f 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -192,11 +192,20 @@ where
 
             let region = self.region(&region_id);
             region.advance().await;
-            let buffer = self
-                .buffers
-                .acquire()
-                .instrument(tracing::debug_span!("acquire_clean_buffer"))
-                .await;
+
+            let buffer = {
+                let timer = self
+                    .metrics
+                    .inner_op_duration_acquire_clean_buffer
+                    .start_timer();
+                let buffer = self
+                    .buffers
+                    .acquire()
+                    .instrument(tracing::debug_span!("acquire_clean_buffer"))
+                    .await;
+                drop(timer);
+                buffer
+            };
             region.attach_buffer(buffer).await;
 
             *current = Some(region.clone());

From 438eec87e90c7a80cb53a06b711c6ea1ad7a0f41 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 11 Oct 2023 06:39:42 -0500
Subject: [PATCH 138/261] refactor: move force to storage writer trait (#172)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |  7 +++++++
 foyer-storage/src/generic.rs    | 21 ++++++---------------
 foyer-storage/src/lazy.rs       |  7 +++++++
 foyer-storage/src/runtime.rs    |  4 ++++
 foyer-storage/src/storage.rs    | 24 +++++++-----------------
 foyer-storage/src/store.rs      | 32 ++++++++++++--------------------
 6 files changed, 43 insertions(+), 52 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 03276da6..01c228b7 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -271,6 +271,13 @@ where
         }
     }
 
+    fn force(&mut self) {
+        match self {
+            BenchStoreWriter::StoreWriter { writer } => writer.force(),
+            BenchStoreWriter::RuntimeStoreWriter { writer } => writer.force(),
+        }
+    }
+
     async fn finish(self, value: Self::Value) -> Result<bool> {
         match self {
             BenchStoreWriter::StoreWriter { writer } => writer.finish(value).await,
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index e5429256..7b66ff1e 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -41,7 +41,7 @@ use crate::{
     region::{Region, RegionHeader, RegionId, REGION_MAGIC},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
-    storage::{ForceStorageWriter, Storage, StorageWriter},
+    storage::{Storage, StorageWriter},
 };
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::core::adapter::Link;
@@ -672,7 +672,7 @@ where
         store.apply_writer(self, value).await
     }
 
-    pub fn set_force(&mut self) {
+    pub fn force(&mut self) {
         self.judges.set_mask(Bitmap::new());
     }
 
@@ -1005,21 +1005,12 @@ where
         self.judge()
     }
 
-    async fn finish(self, value: Self::Value) -> Result<bool> {
-        self.finish(value).await
+    fn force(&mut self) {
+        self.force()
     }
-}
 
-impl<K, V, D, EP, EL> ForceStorageWriter for GenericStoreWriter<K, V, D, EP, EL>
-where
-    K: Key,
-    V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
-{
-    fn set_force(&mut self) {
-        self.set_force()
+    async fn finish(self, value: Self::Value) -> Result<bool> {
+        self.finish(value).await
     }
 }
 
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index a6bb7190..b680589f 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -63,6 +63,13 @@ where
         }
     }
 
+    fn force(&mut self) {
+        match self {
+            LazyStorageWriter::Store { writer } => writer.force(),
+            LazyStorageWriter::None { writer } => writer.force(),
+        }
+    }
+
     async fn finish(self, value: Self::Value) -> Result<bool> {
         match self {
             LazyStorageWriter::Store { writer } => writer.finish(value).await,
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 5490585c..78d0b4bf 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -89,6 +89,10 @@ where
         self.writer.judge()
     }
 
+    fn force(&mut self) {
+        self.writer.force()
+    }
+
     async fn finish(self, value: Self::Value) -> Result<bool> {
         self.runtime
             .spawn(async move { self.writer.finish(value).await })
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 04e3b740..64eb7596 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -31,6 +31,8 @@ pub trait StorageWriter: Send + Sync + Debug {
 
     fn judge(&mut self) -> bool;
 
+    fn force(&mut self);
+
     fn finish(self, value: Self::Value) -> impl Future<Output = Result<bool>> + Send;
 }
 
@@ -208,14 +210,7 @@ pub trait AsyncStorageExt: Storage {
 
 impl<S: Storage> AsyncStorageExt for S {}
 
-pub trait ForceStorageWriter: StorageWriter {
-    fn set_force(&mut self);
-}
-
-pub trait ForceStorageExt: Storage
-where
-    Self::Writer: ForceStorageWriter,
-{
+pub trait ForceStorageExt: Storage {
     #[tracing::instrument(skip(self, value))]
     fn insert_force(
         &self,
@@ -224,7 +219,7 @@ where
     ) -> impl Future<Output = Result<bool>> + Send {
         let weight = key.serialized_len() + value.serialized_len();
         let mut writer = self.writer(key, weight);
-        writer.set_force();
+        writer.force();
         writer.finish(value)
     }
 
@@ -246,7 +241,7 @@ where
     {
         async move {
             let mut writer = self.writer(key, weight);
-            writer.set_force();
+            writer.force();
             if !writer.judge() {
                 return Ok(false);
             }
@@ -281,7 +276,7 @@ where
     {
         async move {
             let mut writer = self.writer(key, weight);
-            writer.set_force();
+            writer.force();
             if !writer.judge() {
                 return Ok(false);
             }
@@ -298,9 +293,4 @@ where
     }
 }
 
-impl<S> ForceStorageExt for S
-where
-    S: Storage,
-    S::Writer: ForceStorageWriter,
-{
-}
+impl<S> ForceStorageExt for S where S: Storage {}
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index e27b1e24..6d694616 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -26,7 +26,7 @@ use crate::{
     error::Result,
     generic::{GenericStore, GenericStoreConfig, GenericStoreWriter},
     region_manager::RegionEpItemAdapter,
-    storage::{ForceStorageWriter, Storage, StorageWriter},
+    storage::{Storage, StorageWriter},
 };
 
 pub type LruFsStore<K, V> =
@@ -89,15 +89,13 @@ impl<K: Key, V: Value> StorageWriter for NoneStoreWriter<K, V> {
         false
     }
 
+    fn force(&mut self) {}
+
     async fn finish(self, _: Self::Value) -> Result<bool> {
         Ok(false)
     }
 }
 
-impl<K: Key, V: Value> ForceStorageWriter for NoneStoreWriter<K, V> {
-    fn set_force(&mut self) {}
-}
-
 #[derive(Debug)]
 pub struct NoneStore<K: Key, V: Value>(PhantomData<(K, V)>);
 
@@ -338,6 +336,15 @@ where
         }
     }
 
+    fn force(&mut self) {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.force(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.force(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.force(),
+            StoreWriter::NoneStoreWriter { writer } => writer.force(),
+        }
+    }
+
     async fn finish(self, value: Self::Value) -> Result<bool> {
         match self {
             StoreWriter::LruFsStorWriter { writer } => writer.finish(value).await,
@@ -348,21 +355,6 @@ where
     }
 }
 
-impl<K, V> ForceStorageWriter for StoreWriter<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn set_force(&mut self) {
-        match self {
-            StoreWriter::LruFsStorWriter { writer } => writer.set_force(),
-            StoreWriter::LfuFsStorWriter { writer } => writer.set_force(),
-            StoreWriter::FifoFsStoreWriter { writer } => writer.set_force(),
-            StoreWriter::NoneStoreWriter { writer } => writer.set_force(),
-        }
-    }
-}
-
 impl<K, V> Storage for Store<K, V>
 where
     K: Key,

From cb6e7b5a78da3008764df720f1e5cc121e11bc67 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 11 Oct 2023 23:28:41 -0500
Subject: [PATCH 139/261] chore: remove unused code (#173)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/region.rs         | 14 ++++++--------
 foyer-storage/src/region_manager.rs |  3 +--
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 1a22d2e5..f0dd37e8 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -37,16 +37,14 @@ pub type Version = u32;
 #[derive(Debug)]
 pub enum AllocateResult {
     Ok(WriteSlice),
-    Full { slice: WriteSlice, remain: usize },
-    None,
+    NotEnough(WriteSlice),
 }
 
 impl AllocateResult {
     pub fn unwrap(self) -> WriteSlice {
         match self {
             AllocateResult::Ok(slice) => slice,
-            AllocateResult::Full { .. } => unreachable!(),
-            AllocateResult::None => unreachable!(),
+            AllocateResult::NotEnough { .. } => unreachable!(),
         }
     }
 }
@@ -155,6 +153,8 @@ where
         }
     }
 
+    /// If there is enough buffer, return `AllocateResult::Ok(slice)``.
+    /// Else, return `AllocateResult::NotEnough(slice)`. `slice` is the remaining buffer.
     #[tracing::instrument(skip(self))]
     pub fn allocate(&self, size: usize) -> AllocateResult {
         let cleanup = {
@@ -174,12 +174,10 @@ where
         let region_id = self.id;
 
         if inner.len + size > inner.capacity {
-            let remain = self.device.region_size() - inner.len;
             inner.len = self.device.region_size();
-            let range = inner.len - self.device.align()..inner.len;
 
             let buffer = inner.buffer.as_mut().unwrap();
-            let slice = unsafe { SliceMut::new(&mut buffer[range]) };
+            let slice = unsafe { SliceMut::new(&mut buffer[offset..]) };
 
             let slice = WriteSlice {
                 slice,
@@ -188,7 +186,7 @@ where
                 offset,
                 cleanup: Some(cleanup),
             };
-            AllocateResult::Full { slice, remain }
+            AllocateResult::NotEnough(slice)
         } else {
             inner.len += size;
 
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 7f09bf4f..a80bf89d 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -163,11 +163,10 @@ where
             if let Some(region) = current.as_ref() {
                 match region.allocate(size) {
                     AllocateResult::Ok(slice) => return Some(slice),
-                    AllocateResult::Full { .. } => {
+                    AllocateResult::NotEnough { .. } => {
                         self.dirty_regions.release(region.id());
                         *current = None;
                     }
-                    AllocateResult::None => {}
                 }
             }
 

From a42965ff83fd9b14a3a0ca121f63b87cf1d2919d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 12 Oct 2023 01:58:32 -0500
Subject: [PATCH 140/261] refactor: refine devcie interface with buffer (#174)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/device/fs.rs  | 42 ++++++++++++++-------------------
 foyer-storage/src/device/mod.rs | 38 ++++++++++++++++-------------
 foyer-storage/src/flusher.rs    |  5 ++--
 foyer-storage/src/reclaimer.rs  |  3 ++-
 foyer-storage/src/region.rs     |  6 ++---
 5 files changed, 48 insertions(+), 46 deletions(-)

diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 539394ee..84762619 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -87,7 +87,7 @@ impl Device for FsDevice {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> DeviceResult<usize> {
+    ) -> (DeviceResult<usize>, impl IoBuf) {
         let file_capacity = self.inner.config.file_capacity;
         assert!(
             offset as usize + len <= file_capacity,
@@ -96,14 +96,13 @@ impl Device for FsDevice {
 
         let fd = self.fd(region);
 
-        let res = asyncify(move || {
+        asyncify(move || {
             let fd = unsafe { BorrowedFd::borrow_raw(fd) };
-            let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[..len], offset as i64)?;
-            Ok(res)
+            let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[..len], offset as i64)
+                .map_err(DeviceError::from);
+            (res, buf)
         })
-        .await?;
-
-        Ok(res)
+        .await
     }
 
     async fn read(
@@ -112,7 +111,7 @@ impl Device for FsDevice {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> DeviceResult<usize> {
+    ) -> (DeviceResult<usize>, impl IoBufMut) {
         let file_capacity = self.inner.config.file_capacity;
         assert!(
             offset as usize + len <= file_capacity,
@@ -121,14 +120,13 @@ impl Device for FsDevice {
 
         let fd = self.fd(region);
 
-        let res = asyncify(move || {
+        asyncify(move || {
             let fd = unsafe { BorrowedFd::borrow_raw(fd) };
-            let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[..len], offset as i64)?;
-            Ok(res)
+            let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[..len], offset as i64)
+                .map_err(DeviceError::from);
+            (res, buf)
         })
-        .await?;
-
-        Ok(res)
+        .await
     }
 
     #[cfg(target_os = "linux")]
@@ -137,14 +135,9 @@ impl Device for FsDevice {
         // Commit fs cache to disk. Linux waits for I/O completions.
         //
         // See also [syncfs(2)](https://man7.org/linux/man-pages/man2/sync.2.html)
-        asyncify(move || {
-            nix::unistd::syncfs(fd)?;
-            Ok(())
-        })
-        .await?;
+        asyncify(move || nix::unistd::syncfs(fd).map_err(DeviceError::from)).await?;
 
         // TODO(MrCroxx): track dirty files and call fsync(2) on them on other target os.
-
         Ok(())
     }
 
@@ -194,8 +187,7 @@ impl FsDevice {
         let path = config.dir.clone();
         let dir = asyncify(move || {
             create_dir_all(&path)?;
-            let dir = File::open(&path)?;
-            Ok(dir)
+            File::open(&path).map_err(DeviceError::from)
         })
         .await?;
 
@@ -278,8 +270,10 @@ mod tests {
         let wbuf = unsafe { Slice::new(&wbuffer) };
         let rbuf = unsafe { SliceMut::new(&mut rbuffer) };
 
-        dev.write(wbuf, 0, 0, ALIGN).await.unwrap();
-        dev.read(rbuf, 0, 0, ALIGN).await.unwrap();
+        let (res, _wbuf) = dev.write(wbuf, 0, 0, ALIGN).await;
+        res.unwrap();
+        let (res, _rbuf) = dev.read(rbuf, 0, 0, ALIGN).await;
+        res.unwrap();
 
         assert_eq!(&wbuffer, &rbuffer);
 
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index dc0ef19e..1f4839f3 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -40,7 +40,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> impl Future<Output = DeviceResult<usize>> + Send;
+    ) -> impl Future<Output = (DeviceResult<usize>, impl IoBuf)> + Send;
 
     #[must_use]
     fn read(
@@ -49,7 +49,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
         region: RegionId,
         offset: u64,
         len: usize,
-    ) -> impl Future<Output = DeviceResult<usize>> + Send;
+    ) -> impl Future<Output = (DeviceResult<usize>, impl IoBufMut)> + Send;
 
     #[must_use]
     fn flush(&self) -> impl Future<Output = DeviceResult<()>> + Send;
@@ -72,21 +72,27 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     }
 }
 
+#[cfg(not(madsim))]
 #[tracing::instrument(level = "trace", skip(f))]
-async fn asyncify<F, T>(f: F) -> DeviceResult<T>
+async fn asyncify<F, T>(f: F) -> T
 where
-    F: FnOnce() -> DeviceResult<T> + Send + 'static,
+    F: FnOnce() -> T + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::task::spawn_blocking(f).await.unwrap()
+}
+
+#[cfg(madsim)]
+#[tracing::instrument(level = "trace", skip(f))]
+async fn asyncify<F, T>(f: F) -> T
+where
+    F: FnOnce() -> T + Send + 'static,
     T: Send + 'static,
 {
-    #[cfg(not(madsim))]
-    match tokio::task::spawn_blocking(f).await {
-        Ok(res) => res,
-        Err(e) => Err(format!("background task failed: {:?}", e,).into()),
-    }
-    #[cfg(madsim)]
     f()
 }
 
+#[cfg(madsim)]
 #[cfg(test)]
 pub mod tests {
     use super::{allocator::AlignedAllocator, *};
@@ -110,22 +116,22 @@ pub mod tests {
 
         async fn write(
             &self,
-            _buf: impl IoBuf,
+            buf: impl IoBuf,
             _region: RegionId,
             _offset: u64,
             _len: usize,
-        ) -> DeviceResult<usize> {
-            Ok(0)
+        ) -> (DeviceResult<usize>, impl IoBuf) {
+            (Ok(0), buf)
         }
 
         async fn read(
             &self,
-            _buf: impl IoBufMut,
+            buf: impl IoBufMut,
             _region: RegionId,
             _offset: u64,
             _len: usize,
-        ) -> DeviceResult<usize> {
-            Ok(0)
+        ) -> (DeviceResult<usize>, impl IoBufMut) {
+            (Ok(0), buf)
         }
 
         async fn flush(&self) -> DeviceResult<()> {
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 72ebf381..c398ca50 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -108,10 +108,11 @@ where
             if let Some(limiter) = &self.rate_limiter && let Some(duration) = limiter.consume(len as f64) {
                 tokio::time::sleep(duration).await;
             }
-            region
+            let (res, _s) = region
                 .device()
                 .write(s, region.id(), offset as u64, len)
-                .await?;
+                .await;
+            res?;
             offset += len;
         }
         drop(slice);
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 6507e2d5..b657cac1 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -213,7 +213,8 @@ where
         let align = region.device().align();
         let mut buf = region.device().io_buffer(align, align);
         (&mut buf[..]).put_slice(&vec![0; align]);
-        region.device().write(buf, region_id, 0, align).await?;
+        let (res, _buf) = region.device().write(buf, region_id, 0, align).await;
+        res?;
 
         // step 4: send clean region
         self.region_manager.clean_regions().release(region_id);
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index f0dd37e8..15cf903a 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -297,11 +297,11 @@ where
                 start + offset + len
             );
             let s = unsafe { SliceMut::new(&mut buf[offset..offset + len]) };
-            let read = match self
+            let (res, _s) = self
                 .device
                 .read(s, region, (start + offset) as u64, len)
-                .await
-            {
+                .await;
+            let read = match res {
                 Ok(bytes) => bytes,
                 Err(e) => {
                     let mut inner = self.inner.write();

From d0d1dbcaf2f7459234bc06919dec96384927daf5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 12 Oct 2023 03:48:43 -0500
Subject: [PATCH 141/261] feat: introduce RangeBoundsExt and refactor device
 interface (#175)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/lib.rs         |   2 +
 foyer-common/src/range.rs       | 117 ++++++++++++++++++++++++++++++++
 foyer-storage/src/device/fs.rs  |  23 +++++--
 foyer-storage/src/device/mod.rs |  11 +--
 foyer-storage/src/flusher.rs    |   2 +-
 foyer-storage/src/reclaimer.rs  |   2 +-
 foyer-storage/src/region.rs     |   2 +-
 7 files changed, 144 insertions(+), 15 deletions(-)
 create mode 100644 foyer-common/src/range.rs

diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 3e46aefc..0b6b776f 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -14,6 +14,7 @@
 
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
+#![feature(bound_map)]
 #![cfg_attr(coverage_nightly, feature(no_coverage))]
 
 pub mod batch;
@@ -21,6 +22,7 @@ pub mod bits;
 pub mod code;
 pub mod erwlock;
 pub mod queue;
+pub mod range;
 pub mod rate;
 pub mod rated_random;
 pub mod rated_ticket;
diff --git a/foyer-common/src/range.rs b/foyer-common/src/range.rs
new file mode 100644
index 00000000..b01e35f7
--- /dev/null
+++ b/foyer-common/src/range.rs
@@ -0,0 +1,117 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::ops::{Add, Bound, Range, RangeBounds, Sub};
+
+mod private {
+
+    pub trait ZeroOne {
+        fn zero() -> Self;
+        fn one() -> Self;
+    }
+
+    macro_rules! impl_one {
+        ($($t:ty),*) => {
+            $(
+                impl ZeroOne for $t {
+                    fn zero() -> Self {
+                        0 as $t
+                    }
+
+                    fn one() -> Self {
+                        1 as $t
+                    }
+                }
+            )*
+        };
+    }
+
+    macro_rules! for_all_num_type {
+        ($macro:ident) => {
+            $macro! { u8, u16, u32, u64, usize, i8, i16, i32, i64, isize, f32, f64 }
+        };
+    }
+
+    for_all_num_type! { impl_one }
+}
+
+use private::ZeroOne;
+
+pub trait Idx = PartialOrd<Self>
+    + Add<Output = Self>
+    + Sub<Output = Self>
+    + Clone
+    + Copy
+    + Send
+    + Sync
+    + 'static
+    + ZeroOne;
+
+pub trait RangeBoundsExt<T: Idx>: RangeBounds<T> {
+    fn start(&self) -> Option<T> {
+        match self.start_bound() {
+            Bound::Included(v) => Some(*v),
+            Bound::Excluded(v) => Some(*v + ZeroOne::one()),
+            Bound::Unbounded => None,
+        }
+    }
+
+    fn end(&self) -> Option<T> {
+        match self.end_bound() {
+            Bound::Included(v) => Some(*v + ZeroOne::one()),
+            Bound::Excluded(v) => Some(*v),
+            Bound::Unbounded => None,
+        }
+    }
+
+    fn start_with_bound(&self, bound: T) -> T {
+        self.start().unwrap_or(bound)
+    }
+
+    fn end_with_bound(&self, bound: T) -> T {
+        self.end().unwrap_or(bound)
+    }
+
+    fn bounds(&self, range: Range<T>) -> Range<T> {
+        let start = self.start_with_bound(range.start);
+        let end = self.end_with_bound(range.end);
+        start..end
+    }
+
+    fn len(&self) -> Option<T> {
+        let start = self.start()?;
+        let end = self.end()?;
+        Some(end - start)
+    }
+
+    fn is_empty(&self) -> bool {
+        match self.len() {
+            Some(len) => len == ZeroOne::zero(),
+            None => false,
+        }
+    }
+
+    fn is_full(&self) -> bool {
+        self.start_bound() == Bound::Unbounded && self.end_bound() == Bound::Unbounded
+    }
+
+    fn map<F, R>(&self, f: F) -> (Bound<R>, Bound<R>)
+    where
+        F: Fn(&T) -> R,
+    {
+        (self.start_bound().map(&f), self.end_bound().map(&f))
+    }
+}
+
+impl<T: Idx, RB: RangeBounds<T>> RangeBoundsExt<T> for RB {}
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 84762619..7baa14ae 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -25,8 +25,9 @@ use super::{
     allocator::AlignedAllocator,
     asyncify,
     error::{DeviceError, DeviceResult},
-    Device, IoBuf, IoBufMut,
+    Device, IoBuf, IoBufMut, IoRange,
 };
+use foyer_common::range::RangeBoundsExt;
 use futures::future::try_join_all;
 use itertools::Itertools;
 
@@ -84,11 +85,15 @@ impl Device for FsDevice {
     async fn write(
         &self,
         buf: impl IoBuf,
+        range: impl IoRange,
         region: RegionId,
         offset: u64,
-        len: usize,
     ) -> (DeviceResult<usize>, impl IoBuf) {
         let file_capacity = self.inner.config.file_capacity;
+
+        let range = range.bounds(0..buf.as_ref().len());
+        let len = RangeBoundsExt::len(&range).unwrap();
+
         assert!(
             offset as usize + len <= file_capacity,
             "offset ({offset}) + len ({len}) <= file capacity ({file_capacity})"
@@ -98,7 +103,7 @@ impl Device for FsDevice {
 
         asyncify(move || {
             let fd = unsafe { BorrowedFd::borrow_raw(fd) };
-            let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[..len], offset as i64)
+            let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[range], offset as i64)
                 .map_err(DeviceError::from);
             (res, buf)
         })
@@ -108,11 +113,15 @@ impl Device for FsDevice {
     async fn read(
         &self,
         mut buf: impl IoBufMut,
+        range: impl IoRange,
         region: RegionId,
         offset: u64,
-        len: usize,
     ) -> (DeviceResult<usize>, impl IoBufMut) {
         let file_capacity = self.inner.config.file_capacity;
+
+        let range = range.bounds(0..buf.as_ref().len());
+        let len = RangeBoundsExt::len(&range).unwrap();
+
         assert!(
             offset as usize + len <= file_capacity,
             "offset ({offset}) + len ({len}) <= file capacity ({file_capacity})"
@@ -122,7 +131,7 @@ impl Device for FsDevice {
 
         asyncify(move || {
             let fd = unsafe { BorrowedFd::borrow_raw(fd) };
-            let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[..len], offset as i64)
+            let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[range], offset as i64)
                 .map_err(DeviceError::from);
             (res, buf)
         })
@@ -270,9 +279,9 @@ mod tests {
         let wbuf = unsafe { Slice::new(&wbuffer) };
         let rbuf = unsafe { SliceMut::new(&mut rbuffer) };
 
-        let (res, _wbuf) = dev.write(wbuf, 0, 0, ALIGN).await;
+        let (res, _wbuf) = dev.write(wbuf, .., 0, 0).await;
         res.unwrap();
-        let (res, _rbuf) = dev.read(rbuf, 0, 0, ALIGN).await;
+        let (res, _rbuf) = dev.read(rbuf, .., 0, 0).await;
         res.unwrap();
 
         assert_eq!(&wbuffer, &rbuffer);
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 1f4839f3..a2d78a92 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -20,11 +20,13 @@ use std::{alloc::Allocator, fmt::Debug};
 
 use crate::region::RegionId;
 use error::DeviceResult;
+use foyer_common::range::RangeBoundsExt;
 use futures::Future;
 
 pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
 pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
 pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
+pub trait IoRange = RangeBoundsExt<usize> + Send + Sync + 'static + Debug;
 
 pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     type IoBufferAllocator: BufferAllocator;
@@ -37,18 +39,18 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     fn write(
         &self,
         buf: impl IoBuf,
+        range: impl IoRange,
         region: RegionId,
         offset: u64,
-        len: usize,
     ) -> impl Future<Output = (DeviceResult<usize>, impl IoBuf)> + Send;
 
     #[must_use]
     fn read(
         &self,
         buf: impl IoBufMut,
+        range: impl IoRange,
         region: RegionId,
         offset: u64,
-        len: usize,
     ) -> impl Future<Output = (DeviceResult<usize>, impl IoBufMut)> + Send;
 
     #[must_use]
@@ -92,7 +94,6 @@ where
     f()
 }
 
-#[cfg(madsim)]
 #[cfg(test)]
 pub mod tests {
     use super::{allocator::AlignedAllocator, *};
@@ -117,9 +118,9 @@ pub mod tests {
         async fn write(
             &self,
             buf: impl IoBuf,
+            _range: impl IoRange,
             _region: RegionId,
             _offset: u64,
-            _len: usize,
         ) -> (DeviceResult<usize>, impl IoBuf) {
             (Ok(0), buf)
         }
@@ -127,9 +128,9 @@ pub mod tests {
         async fn read(
             &self,
             buf: impl IoBufMut,
+            _range: impl IoRange,
             _region: RegionId,
             _offset: u64,
-            _len: usize,
         ) -> (DeviceResult<usize>, impl IoBufMut) {
             (Ok(0), buf)
         }
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index c398ca50..927f4f9d 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -110,7 +110,7 @@ where
             }
             let (res, _s) = region
                 .device()
-                .write(s, region.id(), offset as u64, len)
+                .write(s, .., region.id(), offset as u64)
                 .await;
             res?;
             offset += len;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index b657cac1..f6f70f8b 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -213,7 +213,7 @@ where
         let align = region.device().align();
         let mut buf = region.device().io_buffer(align, align);
         (&mut buf[..]).put_slice(&vec![0; align]);
-        let (res, _buf) = region.device().write(buf, region_id, 0, align).await;
+        let (res, _buf) = region.device().write(buf, .., region_id, 0).await;
         res?;
 
         // step 4: send clean region
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 15cf903a..91a106ef 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -299,7 +299,7 @@ where
             let s = unsafe { SliceMut::new(&mut buf[offset..offset + len]) };
             let (res, _s) = self
                 .device
-                .read(s, region, (start + offset) as u64, len)
+                .read(s, .., region, (start + offset) as u64)
                 .await;
             let read = match res {
                 Ok(bytes) => bytes,

From f2db3098ceaad7a99fb5fc7fc31dc7cf4acb2396 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 13 Oct 2023 05:45:19 +0000
Subject: [PATCH 142/261] chore(deps): update console-subscriber requirement
 from 0.1 to 0.2 (#147)

Updates the requirements on [console-subscriber](https://github.com/tokio-rs/console) to permit the latest version.
- [Release notes](https://github.com/tokio-rs/console/releases)
- [Commits](https://github.com/tokio-rs/console/compare/console-subscriber-v0.1.0...console-subscriber-v0.2.0)

---
updated-dependencies:
- dependency-name: console-subscriber
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 foyer-intrusive/Cargo.toml      | 2 +-
 foyer-storage-bench/Cargo.toml  | 2 +-
 foyer-storage/Cargo.toml        | 2 +-
 foyer-workspace-hack/Cargo.toml | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 75eaad91..412e58db 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -15,7 +15,7 @@ bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
-itertools = "0.10.5"
+itertools = "0.11"
 memoffset = "0.9"
 parking_lot = "0.12"
 paste = "1.0"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 30c1a747..54ecf813 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -14,7 +14,7 @@ normal = ["foyer-workspace-hack"]
 anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
-console-subscriber = { version = "0.1", optional = true }
+console-subscriber = { version = "0.2", optional = true }
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-storage = { path = "../foyer-storage" }
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index e98544a2..86f801b0 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -21,7 +21,7 @@ foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
-itertools = "0.11.0"
+itertools = "0.11"
 libc = "0.2"
 memoffset = "0.9"
 nix = { version = "0.27", features = ["fs", "mman", "uio"] }
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index bf3e6df1..d9926347 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -20,7 +20,7 @@ futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
 hyper = { version = "0.14", features = ["full"] }
-itertools = { version = "0.10" }
+itertools = { version = "0.11" }
 libc = { version = "0.2", features = ["extra_traits"] }
 memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
@@ -35,7 +35,7 @@ tracing-core = { version = "0.1" }
 
 [build-dependencies]
 either = { version = "1", default-features = false, features = ["use_std"] }
-itertools = { version = "0.10" }
+itertools = { version = "0.11" }
 proc-macro2 = { version = "1" }
 quote = { version = "1" }
 syn = { version = "2", features = ["extra-traits", "full", "visit-mut"] }

From 691f84fafc9e2bde9010e51ddf7a572eaf658eb6 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 15 Oct 2023 10:11:55 -0500
Subject: [PATCH 143/261] refactor: remove unnecessary unsafe silce usage
 (#176)

* modify device interface

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refine test

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* remove comment

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/device/fs.rs  | 27 +++++++++++++------------
 foyer-storage/src/device/mod.rs | 36 +++++++++++++++++++++------------
 foyer-storage/src/flusher.rs    | 14 +++++++++----
 3 files changed, 47 insertions(+), 30 deletions(-)

diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 7baa14ae..e042c08a 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -82,13 +82,16 @@ impl Device for FsDevice {
         Self::open(config).await
     }
 
-    async fn write(
+    async fn write<B>(
         &self,
-        buf: impl IoBuf,
+        buf: B,
         range: impl IoRange,
         region: RegionId,
         offset: u64,
-    ) -> (DeviceResult<usize>, impl IoBuf) {
+    ) -> (DeviceResult<usize>, B)
+    where
+        B: IoBuf,
+    {
         let file_capacity = self.inner.config.file_capacity;
 
         let range = range.bounds(0..buf.as_ref().len());
@@ -110,13 +113,16 @@ impl Device for FsDevice {
         .await
     }
 
-    async fn read(
+    async fn read<B>(
         &self,
-        mut buf: impl IoBufMut,
+        mut buf: B,
         range: impl IoRange,
         region: RegionId,
         offset: u64,
-    ) -> (DeviceResult<usize>, impl IoBufMut) {
+    ) -> (DeviceResult<usize>, B)
+    where
+        B: IoBufMut,
+    {
         let file_capacity = self.inner.config.file_capacity;
 
         let range = range.bounds(0..buf.as_ref().len());
@@ -250,8 +256,6 @@ mod tests {
 
     use bytes::BufMut;
 
-    use crate::slice::{Slice, SliceMut};
-
     use super::*;
 
     const FILES: usize = 8;
@@ -276,12 +280,9 @@ mod tests {
         let mut rbuffer = dev.io_buffer(ALIGN, ALIGN);
         (&mut rbuffer[..]).put_slice(&[0; ALIGN]);
 
-        let wbuf = unsafe { Slice::new(&wbuffer) };
-        let rbuf = unsafe { SliceMut::new(&mut rbuffer) };
-
-        let (res, _wbuf) = dev.write(wbuf, .., 0, 0).await;
+        let (res, wbuffer) = dev.write(wbuffer, .., 0, 0).await;
         res.unwrap();
-        let (res, _rbuf) = dev.read(rbuf, .., 0, 0).await;
+        let (res, rbuffer) = dev.read(rbuffer, .., 0, 0).await;
         res.unwrap();
 
         assert_eq!(&wbuffer, &rbuffer);
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index a2d78a92..3780ae2e 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -26,7 +26,7 @@ use futures::Future;
 pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
 pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
 pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
-pub trait IoRange = RangeBoundsExt<usize> + Send + Sync + 'static + Debug;
+pub trait IoRange = RangeBoundsExt<usize> + Sized + Send + Sync + 'static + Debug;
 
 pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     type IoBufferAllocator: BufferAllocator;
@@ -36,22 +36,26 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     fn open(config: Self::Config) -> impl Future<Output = DeviceResult<Self>> + Send;
 
     #[must_use]
-    fn write(
+    fn write<B>(
         &self,
-        buf: impl IoBuf,
+        buf: B,
         range: impl IoRange,
         region: RegionId,
         offset: u64,
-    ) -> impl Future<Output = (DeviceResult<usize>, impl IoBuf)> + Send;
+    ) -> impl Future<Output = (DeviceResult<usize>, B)> + Send
+    where
+        B: IoBuf;
 
     #[must_use]
-    fn read(
+    fn read<B>(
         &self,
-        buf: impl IoBufMut,
+        buf: B,
         range: impl IoRange,
         region: RegionId,
         offset: u64,
-    ) -> impl Future<Output = (DeviceResult<usize>, impl IoBufMut)> + Send;
+    ) -> impl Future<Output = (DeviceResult<usize>, B)> + Send
+    where
+        B: IoBufMut;
 
     #[must_use]
     fn flush(&self) -> impl Future<Output = DeviceResult<()>> + Send;
@@ -115,23 +119,29 @@ pub mod tests {
             Ok(Self::new(config))
         }
 
-        async fn write(
+        async fn write<B>(
             &self,
-            buf: impl IoBuf,
+            buf: B,
             _range: impl IoRange,
             _region: RegionId,
             _offset: u64,
-        ) -> (DeviceResult<usize>, impl IoBuf) {
+        ) -> (DeviceResult<usize>, B)
+        where
+            B: IoBuf,
+        {
             (Ok(0), buf)
         }
 
-        async fn read(
+        async fn read<B>(
             &self,
-            buf: impl IoBufMut,
+            buf: B,
             _range: impl IoRange,
             _region: RegionId,
             _offset: u64,
-        ) -> (DeviceResult<usize>, impl IoBufMut) {
+        ) -> (DeviceResult<usize>, B)
+        where
+            B: IoBufMut,
+        {
             (Ok(0), buf)
         }
 
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 927f4f9d..f8d13682 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -25,7 +25,6 @@ use crate::{
     metrics::Metrics,
     region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
-    slice::Slice,
 };
 
 #[derive(Debug)]
@@ -90,6 +89,7 @@ where
 
         // step 1: write buffer back to device
         let slice = region.load(.., 0).await?.unwrap();
+        let mut slice = Some(slice);
 
         {
             // wait all physical readers (from previous version) and writers done
@@ -104,17 +104,23 @@ where
             let start = offset;
             let end = std::cmp::min(offset + len, region.device().region_size());
 
-            let s = unsafe { Slice::new(&slice.as_ref()[start..end]) };
             if let Some(limiter) = &self.rate_limiter && let Some(duration) = limiter.consume(len as f64) {
                 tokio::time::sleep(duration).await;
             }
-            let (res, _s) = region
+            let (res, s) = region
                 .device()
-                .write(s, .., region.id(), offset as u64)
+                .write(
+                    slice.take().unwrap(),
+                    start..end,
+                    region.id(),
+                    offset as u64,
+                )
                 .await;
             res?;
+            slice = Some(s);
             offset += len;
         }
+
         drop(slice);
 
         tracing::trace!("[flusher] step 2");

From 3490e3c1b692ac41f57365b70b06b470b2661cc9 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 16 Oct 2023 01:36:04 -0500
Subject: [PATCH 144/261] refactor: code api receives Buf and BufMut to support
 non-continuous buf (#177)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/code.rs | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index d73dfa41..55039ece 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -40,12 +40,12 @@ pub trait Key:
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn write(&self, buf: &mut [u8]) {
+    fn write(&self, buf: impl BufMut) {
         panic!("Method `write` must be implemented for `Key` if storage is used.")
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn read(buf: &[u8]) -> Self {
+    fn read(buf: impl Buf) -> Self {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
 }
@@ -63,12 +63,12 @@ pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn write(&self, buf: &mut [u8]) {
+    fn write(&self, buf: impl BufMut) {
         panic!("Method `write` must be implemented for `Value` if storage is used.")
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn read(buf: &[u8]) -> Self {
+    fn read(buf: impl Buf) -> Self {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
 }
@@ -93,12 +93,12 @@ macro_rules! impl_key {
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn write(&self, mut buf: &mut [u8]) {
+                    fn write(&self, mut buf: impl BufMut) {
                         buf.[< put_ $type>](*self)
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn read(mut buf: &[u8]) -> Self {
+                    fn read(mut buf: impl Buf) -> Self {
                         buf.[< get_ $type>]()
                     }
                 }
@@ -118,12 +118,12 @@ macro_rules! impl_value {
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn write(&self, mut buf: &mut [u8]) {
+                    fn write(&self, mut buf: impl BufMut) {
                         buf.[< put_ $type>](*self)
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn read(mut buf: &[u8]) -> Self {
+                    fn read(mut buf: impl Buf) -> Self {
                         buf.[< get_ $type>]()
                     }
                 }
@@ -147,12 +147,17 @@ impl Value for Vec<u8> {
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn write(&self, mut buf: &mut [u8]) {
+    fn write(&self, mut buf: impl BufMut) {
         buf.put_slice(self);
     }
 
+    #[expect(clippy::uninit_vec)]
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn read(buf: &[u8]) -> Self {
-        buf.to_vec()
+    fn read(mut buf: impl Buf) -> Self {
+        let bytes = buf.remaining();
+        let mut v = Vec::with_capacity(bytes);
+        unsafe { v.set_len(bytes) };
+        buf.copy_to_slice(&mut v);
+        v
     }
 }

From 6dec2f4749e16a887cebf5d650283aa959dedd1f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 17 Oct 2023 02:54:41 -0500
Subject: [PATCH 145/261] Revert "refactor: code api receives Buf and BufMut to
 support non-continuous buf (#177)" (#178)

This reverts commit 3490e3c1b692ac41f57365b70b06b470b2661cc9.
---
 foyer-common/src/code.rs | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 55039ece..d73dfa41 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -40,12 +40,12 @@ pub trait Key:
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn write(&self, buf: impl BufMut) {
+    fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Key` if storage is used.")
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn read(buf: impl Buf) -> Self {
+    fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
 }
@@ -63,12 +63,12 @@ pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn write(&self, buf: impl BufMut) {
+    fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Value` if storage is used.")
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn read(buf: impl Buf) -> Self {
+    fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
 }
@@ -93,12 +93,12 @@ macro_rules! impl_key {
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn write(&self, mut buf: impl BufMut) {
+                    fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn read(mut buf: impl Buf) -> Self {
+                    fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
                 }
@@ -118,12 +118,12 @@ macro_rules! impl_value {
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn write(&self, mut buf: impl BufMut) {
+                    fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
                     #[cfg_attr(coverage_nightly, no_coverage)]
-                    fn read(mut buf: impl Buf) -> Self {
+                    fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
                 }
@@ -147,17 +147,12 @@ impl Value for Vec<u8> {
     }
 
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn write(&self, mut buf: impl BufMut) {
+    fn write(&self, mut buf: &mut [u8]) {
         buf.put_slice(self);
     }
 
-    #[expect(clippy::uninit_vec)]
     #[cfg_attr(coverage_nightly, no_coverage)]
-    fn read(mut buf: impl Buf) -> Self {
-        let bytes = buf.remaining();
-        let mut v = Vec::with_capacity(bytes);
-        unsafe { v.set_len(bytes) };
-        buf.copy_to_slice(&mut v);
-        v
+    fn read(buf: &[u8]) -> Self {
+        buf.to_vec()
     }
 }

From 338ead43ece87fa3184a345707417a960d1b408c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 18 Oct 2023 04:38:31 -0500
Subject: [PATCH 146/261] refactor: replace indices with catalog for further
 usage (#179)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml                |   2 +-
 foyer-storage-bench/src/main.rs        |   5 +
 foyer-storage/src/admission/mod.rs     |   4 +-
 foyer-storage/src/catalog.rs           | 149 +++++++++++++
 foyer-storage/src/generic.rs           | 298 +++++++++++++++----------
 foyer-storage/src/lazy.rs              |   2 +
 foyer-storage/src/lib.rs               |   2 +-
 foyer-storage/src/reclaimer.rs         |   2 +-
 foyer-storage/src/reinsertion/exist.rs |   6 +-
 foyer-storage/src/reinsertion/mod.rs   |   4 +-
 foyer-storage/src/ring.rs              | 223 ++++++++++++++++++
 foyer-storage/tests/storage_test.rs    |   4 +
 12 files changed, 569 insertions(+), 132 deletions(-)
 create mode 100644 foyer-storage/src/catalog.rs
 create mode 100644 foyer-storage/src/ring.rs

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 2fd9bf0b..93357a95 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -20,4 +20,4 @@ tokio = { workspace = true }
 tracing = "0.1"
 
 [dev-dependencies]
-itertools = "0.11.0"
+itertools = "0.11"
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 01c228b7..ad8c4e91 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -176,6 +176,10 @@ pub struct Args {
     #[arg(long, default_value_t = 0)]
     allocator_bits: usize,
 
+    /// Catalog indices sharding bits.
+    #[arg(long, default_value_t = 6)]
+    catalog_bits: usize,
+
     /// weigher to enable metrics exporter
     #[arg(long, default_value_t = false)]
     metrics: bool,
@@ -549,6 +553,7 @@ async fn main() {
         eviction_config,
         device_config,
         allocator_bits: args.allocator_bits,
+        catalog_bits: args.catalog_bits,
         admissions,
         reinsertions,
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 9c5c8577..e6bae0d5 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -16,14 +16,14 @@ use foyer_common::code::{Key, Value};
 
 use std::{fmt::Debug, sync::Arc};
 
-use crate::{indices::Indices, metrics::Metrics};
+use crate::{catalog::Catalog, metrics::Metrics};
 
 #[expect(unused_variables)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, indices: &Arc<Indices<Self::Key>>) {}
+    fn init(&self, indices: &Arc<Catalog<Self::Key>>) {}
 
     fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
 
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
new file mode 100644
index 00000000..2f8b99b7
--- /dev/null
+++ b/foyer-storage/src/catalog.rs
@@ -0,0 +1,149 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    collections::btree_map::{BTreeMap, Entry},
+    hash::Hasher,
+    sync::Arc,
+};
+
+use foyer_common::code::Key;
+use itertools::Itertools;
+use parking_lot::{Mutex, RwLock};
+use twox_hash::XxHash64;
+
+use crate::region::{RegionId, Version};
+
+pub type Sequence = u64;
+
+#[derive(Debug, Clone)]
+pub enum Index {
+    RingBuffer {},
+    Region {
+        region: RegionId,
+        version: Version,
+        offset: u32,
+        len: u32,
+        key_len: u32,
+        value_len: u32,
+    },
+}
+
+#[derive(Debug, Clone)]
+pub struct IndexInfo {
+    pub sequence: Sequence,
+    pub index: Index,
+}
+
+#[derive(Debug)]
+pub struct Catalog<K>
+where
+    K: Key,
+{
+    /// `items` sharding bits.
+    bits: usize,
+
+    /// Sharded by key hash.
+    infos: Vec<RwLock<BTreeMap<Arc<K>, IndexInfo>>>,
+
+    /// Sharded by region id.
+    regions: Vec<Mutex<BTreeMap<Arc<K>, u64>>>,
+}
+
+impl<K> Catalog<K>
+where
+    K: Key,
+{
+    pub fn new(regions: usize, bits: usize) -> Self {
+        let infos = (0..1 << bits)
+            .map(|_| RwLock::new(BTreeMap::new()))
+            .collect_vec();
+        let regions = (0..regions)
+            .map(|_| Mutex::new(BTreeMap::new()))
+            .collect_vec();
+        Self {
+            bits,
+            infos,
+            regions,
+        }
+    }
+
+    pub fn insert(&self, key: K, info: IndexInfo) {
+        // TODO(MrCroxx): compare sequence.
+        let key = Arc::new(key);
+
+        if let Index::Region { region, .. } = info.index {
+            self.regions[region as usize]
+                .lock()
+                .insert(key.clone(), info.sequence);
+        };
+
+        let shard = self.shard(&key);
+        // TODO(MrCroxx): handle old key?
+        let _ = self.infos[shard].write().insert(key.clone(), info);
+    }
+
+    pub fn lookup(&self, key: &K) -> Option<IndexInfo> {
+        let shard = self.shard(key);
+        self.infos[shard].read().get(key).cloned()
+    }
+
+    pub fn remove(&self, key: &K) -> Option<IndexInfo> {
+        let shard = self.shard(key);
+        let info: Option<IndexInfo> = self.infos[shard].write().remove(key);
+        if let Some(info) = &info && let Index::Region { region,..} = info.index {
+            self.regions[region as usize].lock().remove(key);
+        }
+        info
+    }
+
+    pub fn take_region(&self, region: &RegionId) -> Vec<IndexInfo> {
+        let mut keys = BTreeMap::new();
+        std::mem::swap(&mut *self.regions[*region as usize].lock(), &mut keys);
+
+        let mut infos = Vec::with_capacity(keys.len());
+        for (key, sequence) in keys {
+            let shard = self.shard(&key);
+            match self.infos[shard].write().entry(key.clone()) {
+                Entry::Vacant(_) => continue,
+                Entry::Occupied(o) => {
+                    if o.get().sequence == sequence {
+                        let info = o.remove();
+                        infos.push(info);
+                    }
+                }
+            };
+        }
+        infos
+    }
+
+    pub fn clear(&self) {
+        for shard in self.infos.iter() {
+            shard.write().clear();
+        }
+        for region in self.regions.iter() {
+            region.lock().clear();
+        }
+    }
+
+    fn shard(&self, key: &K) -> usize {
+        self.hash(key) as usize & ((1 << self.bits) - 1)
+    }
+
+    fn hash(&self, key: &K) -> u64 {
+        let mut hasher = XxHash64::default();
+        key.hash(&mut hasher);
+        hasher.finish()
+    }
+}
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 7b66ff1e..7709d1b1 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -15,7 +15,10 @@
 use std::{
     fmt::Debug,
     marker::PhantomData,
-    sync::Arc,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
     time::{Duration, Instant},
 };
 
@@ -31,10 +34,10 @@ use twox_hash::XxHash64;
 
 use crate::{
     admission::AdmissionPolicy,
+    catalog::{Catalog, Index, IndexInfo, Sequence},
     device::Device,
     error::Result,
     flusher::Flusher,
-    indices::{Index, Indices},
     judge::Judges,
     metrics::{Metrics, METRICS},
     reclaimer::Reclaimer,
@@ -73,6 +76,9 @@ where
     ///       (buffer count = buffer pool size / device region size)
     pub allocator_bits: usize,
 
+    /// Catalog indices sharding bits.
+    pub catalog_bits: usize,
+
     /// Admission policies.
     pub admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
 
@@ -118,6 +124,7 @@ where
             .field("eviction_config", &self.eviction_config)
             .field("device_config", &self.device_config)
             .field("allocator_bits", &self.allocator_bits)
+            .field("catalog_bits", &self.catalog_bits)
             .field("admissions", &self.admissions)
             .field("reinsertions", &self.reinsertions)
             .field("buffer_pool_size", &self.buffer_pool_size)
@@ -145,6 +152,7 @@ where
             eviction_config: self.eviction_config.clone(),
             device_config: self.device_config.clone(),
             allocator_bits: self.allocator_bits,
+            catalog_bits: self.catalog_bits,
             admissions: self.admissions.clone(),
             reinsertions: self.reinsertions.clone(),
             buffer_pool_size: self.buffer_pool_size,
@@ -195,7 +203,8 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    indices: Arc<Indices<K>>,
+    sequence: AtomicU64,
+    indices: Arc<Catalog<K>>,
 
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
@@ -249,7 +258,7 @@ where
             metrics.clone(),
         ));
 
-        let indices = Arc::new(Indices::new(device.regions()));
+        let indices = Arc::new(Catalog::new(device.regions(), config.catalog_bits));
 
         let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
         let (reclaimers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
@@ -262,6 +271,7 @@ where
             .collect_vec();
 
         let inner = GenericStoreInner {
+            sequence: AtomicU64::new(0),
             indices: indices.clone(),
             region_manager: region_manager.clone(),
             device: device.clone(),
@@ -319,7 +329,8 @@ where
             })
             .collect_vec();
 
-        store.recover(config.recover_concurrency).await?;
+        let sequence = store.recover(config.recover_concurrency).await?;
+        store.inner.sequence.store(sequence + 1, Ordering::Relaxed);
 
         let flusher_handles = flushers
             .into_iter()
@@ -377,8 +388,8 @@ where
     async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
 
-        let index = match self.inner.indices.lookup(key) {
-            Some(index) => index,
+        let info = match self.inner.indices.lookup(key) {
+            Some(info) => info,
             None => {
                 self.inner
                     .metrics
@@ -388,45 +399,57 @@ where
             }
         };
 
-        self.inner.region_manager.record_access(&index.region);
-        let region = self.inner.region_manager.region(&index.region);
-        let start = index.offset as usize;
-        let end = start + index.len as usize;
+        match info.index {
+            crate::catalog::Index::RingBuffer {} => todo!(),
+            crate::catalog::Index::Region {
+                region,
+                version,
+                offset,
+                len,
+                key_len: _,
+                value_len: _,
+            } => {
+                self.inner.region_manager.record_access(&region);
+                let region = self.inner.region_manager.region(&region);
+                let start = offset as usize;
+                let end = start + len as usize;
+
+                // TODO(MrCroxx): read value only
+                let slice = match region.load(start..end, version).await? {
+                    Some(slice) => slice,
+                    None => {
+                        // Remove index if the storage layer fails to lookup it (because of region version mismatch).
+                        self.inner.indices.remove(key);
+                        self.inner
+                            .metrics
+                            .op_duration_lookup_miss
+                            .observe(now.elapsed().as_secs_f64());
+                        return Ok(None);
+                    }
+                };
+                self.inner
+                    .metrics
+                    .op_bytes_lookup
+                    .inc_by(slice.len() as u64);
+
+                let res = match read_entry::<K, V>(slice.as_ref()) {
+                    Some((_key, value)) => Ok(Some(value)),
+                    None => {
+                        // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
+                        self.inner.indices.remove(key);
+                        Ok(None)
+                    }
+                };
+                drop(slice);
 
-        // TODO(MrCroxx): read value only
-        let slice = match region.load(start..end, index.version).await? {
-            Some(slice) => slice,
-            None => {
-                // Remove index if the storage layer fails to lookup it (because of region version mismatch).
-                self.inner.indices.remove(key);
                 self.inner
                     .metrics
-                    .op_duration_lookup_miss
+                    .op_duration_lookup_hit
                     .observe(now.elapsed().as_secs_f64());
-                return Ok(None);
-            }
-        };
-        self.inner
-            .metrics
-            .op_bytes_lookup
-            .inc_by(slice.len() as u64);
 
-        let res = match read_entry::<K, V>(slice.as_ref()) {
-            Some((_key, value)) => Ok(Some(value)),
-            None => {
-                // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
-                self.inner.indices.remove(key);
-                Ok(None)
+                res
             }
-        };
-        drop(slice);
-
-        self.inner
-            .metrics
-            .op_duration_lookup_hit
-            .observe(now.elapsed().as_secs_f64());
-
-        res
+        }
     }
 
     #[tracing::instrument(skip(self))]
@@ -447,7 +470,7 @@ where
         Ok(())
     }
 
-    pub(crate) fn indices(&self) -> &Arc<Indices<K>> {
+    pub(crate) fn catalog(&self) -> &Arc<Catalog<K>> {
         &self.inner.indices
     }
 
@@ -466,7 +489,7 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    async fn recover(&self, concurrency: usize) -> Result<()> {
+    async fn recover(&self, concurrency: usize) -> Result<Sequence> {
         tracing::info!("start store recovery");
 
         let (tx, rx) = async_channel::bounded(concurrency);
@@ -487,13 +510,15 @@ where
         }
 
         let mut recovered = 0;
+        let mut sequence = 0;
 
         let results = try_join_all(handles).await.map_err(anyhow::Error::from)?;
 
         for (region_id, result) in results.into_iter().enumerate() {
-            if result? {
+            if let Some(seq) = result? {
                 tracing::debug!("region {} is recovered", region_id);
-                recovered += 1
+                recovered += 1;
+                sequence = std::cmp::max(sequence, seq);
             }
         }
 
@@ -508,25 +533,27 @@ where
             self.inner.region_manager.clean_regions().flash();
         }
 
-        Ok(())
+        Ok(sequence)
     }
 
-    /// Return `true` if region is valid, otherwise `false`
+    /// Return `Some(max sequence)` if region is valid, otherwise `None`
     async fn recover_region(
         region_id: RegionId,
         region_manager: Arc<RegionManager<D, EP, EL>>,
-        indices: Arc<Indices<K>>,
-    ) -> Result<bool> {
+        indices: Arc<Catalog<K>>,
+    ) -> Result<Option<Sequence>> {
         let region = region_manager.region(&region_id).clone();
+        let mut sequence = 0;
         let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
-            while let Some(index) = iter.next().await? {
-                indices.insert(index);
+            while let Some((key, info)) = iter.next().await? {
+                sequence = std::cmp::max(sequence, info.sequence);
+                indices.insert(key, info);
             }
             region_manager.eviction_push(region_id);
-            true
+            Some(sequence)
         } else {
             region_manager.clean_regions().release(region_id);
-            false
+            None
         };
         Ok(res)
     }
@@ -553,15 +580,21 @@ where
 
         let now = Instant::now();
 
+        let sequence = if let Some(sequence) = writer.sequence {
+            sequence
+        } else {
+            self.inner.sequence.fetch_add(1, Ordering::Relaxed)
+        };
+
         writer.is_inserted = true;
-        let key = &writer.key;
+        let key = writer.key;
 
         for (i, admission) in self.inner.admissions.iter().enumerate() {
             let judge = writer.judges.get(i);
-            admission.on_insert(key, writer.weight, &self.inner.metrics, judge);
+            admission.on_insert(&key, writer.weight, &self.inner.metrics, judge);
         }
 
-        let serialized_len = self.serialized_len(key, &value);
+        let serialized_len = self.serialized_len(&key, &value);
 
         if key.serialized_len() + value.serialized_len() != writer.weight {
             tracing::error!(
@@ -586,21 +619,22 @@ where
             None => return Ok(false),
         };
 
-        write_entry(slice.as_mut(), key, &value);
-
-        let index = Index {
-            region: slice.region_id(),
-            version: slice.version(),
-            offset: slice.offset() as u32,
-            len: slice.len() as u32,
-            key_len: key.serialized_len() as u32,
-            value_len: value.serialized_len() as u32,
-
-            key: key.clone(),
+        write_entry(slice.as_mut(), &key, &value, sequence);
+
+        let info = IndexInfo {
+            sequence,
+            index: Index::Region {
+                region: slice.region_id(),
+                version: slice.version(),
+                offset: slice.offset() as u32,
+                len: slice.len() as u32,
+                key_len: key.serialized_len() as u32,
+                value_len: value.serialized_len() as u32,
+            },
         };
         drop(slice);
 
-        self.inner.indices.insert(index);
+        self.inner.indices.insert(key, info);
 
         let duration = now.elapsed() + writer.duration;
         self.inner
@@ -624,6 +658,8 @@ where
     key: K,
     weight: usize,
 
+    sequence: Option<Sequence>,
+
     judges: Judges,
     is_judged: bool,
 
@@ -648,6 +684,7 @@ where
             store,
             key,
             weight,
+            sequence: None,
             judges,
             is_judged: false,
             duration: Duration::from_nanos(0),
@@ -683,6 +720,10 @@ where
     pub fn set_skippable(&mut self) {
         self.is_skippable = true
     }
+
+    pub fn set_sequence(&mut self, sequence: Sequence) {
+        self.sequence = Some(sequence);
+    }
 }
 
 impl<K, V, D, EP, EL> Debug for GenericStoreWriter<K, V, D, EP, EL>
@@ -705,45 +746,45 @@ where
     }
 }
 
-impl<K, V, D, EP, EL> Drop for GenericStoreWriter<K, V, D, EP, EL>
-where
-    K: Key,
-    V: Value,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
-{
-    fn drop(&mut self) {
-        if !self.is_inserted {
-            self.store
-                .inner
-                .metrics
-                .op_duration_insert_dropped
-                .observe(self.duration.as_secs_f64());
-            let mut filtered = false;
-            if self.is_judged {
-                for (i, admission) in self.store.inner.admissions.iter().enumerate() {
-                    let judge = self.judges.get(i);
-                    admission.on_drop(&self.key, self.weight, &self.store.inner.metrics, judge);
-                }
-                filtered = !self.judge();
-            }
-            if filtered {
-                self.store
-                    .inner
-                    .metrics
-                    .op_duration_insert_filtered
-                    .observe(self.duration.as_secs_f64());
-            } else {
-                self.store
-                    .inner
-                    .metrics
-                    .op_duration_insert_dropped
-                    .observe(self.duration.as_secs_f64());
-            }
-        }
-    }
-}
+// impl<K, V, D, EP, EL> Drop for GenericStoreWriter<K, V, D, EP, EL>
+// where
+//     K: Key,
+//     V: Value,
+//     D: Device,
+//     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+//     EL: Link,
+// {
+//     fn drop(&mut self) {
+//         if !self.is_inserted {
+//             self.store
+//                 .inner
+//                 .metrics
+//                 .op_duration_insert_dropped
+//                 .observe(self.duration.as_secs_f64());
+//             let mut filtered = false;
+//             if self.is_judged {
+//                 for (i, admission) in self.store.inner.admissions.iter().enumerate() {
+//                     let judge = self.judges.get(i);
+//                     admission.on_drop(&self.key, self.weight, &self.store.inner.metrics, judge);
+//                 }
+//                 filtered = !self.judge();
+//             }
+//             if filtered {
+//                 self.store
+//                     .inner
+//                     .metrics
+//                     .op_duration_insert_filtered
+//                     .observe(self.duration.as_secs_f64());
+//             } else {
+//                 self.store
+//                     .inner
+//                     .metrics
+//                     .op_duration_insert_dropped
+//                     .observe(self.duration.as_secs_f64());
+//             }
+//         }
+//     }
+// }
 
 const ENTRY_MAGIC: u32 = 0x97_00_00_00;
 const ENTRY_MAGIC_MASK: u32 = 0xFF_00_00_00;
@@ -752,17 +793,19 @@ const ENTRY_MAGIC_MASK: u32 = 0xFF_00_00_00;
 struct EntryHeader {
     key_len: u32,
     value_len: u32,
+    sequence: Sequence,
     checksum: u64,
 }
 
 impl EntryHeader {
     fn serialized_len() -> usize {
-        4 + 4 + 8
+        4 + 4 + 8 + 8
     }
 
     fn write(&self, mut buf: &mut [u8]) {
         buf.put_u32(self.key_len | ENTRY_MAGIC);
         buf.put_u32(self.value_len);
+        buf.put_u64(self.sequence);
         buf.put_u64(self.checksum);
     }
 
@@ -776,11 +819,13 @@ impl EntryHeader {
 
         let key_len = head ^ ENTRY_MAGIC;
         let value_len = buf.get_u32();
+        let sequence = buf.get_u64();
         let checksum = buf.get_u64();
 
         Some(Self {
             key_len,
             value_len,
+            sequence,
             checksum,
         })
     }
@@ -791,7 +836,7 @@ impl EntryHeader {
 /// # Safety
 ///
 /// `buf.len()` must excatly fit entry size
-fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V)
+fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V, sequence: Sequence)
 where
     K: Key,
     V: Value,
@@ -806,6 +851,7 @@ where
     let header = EntryHeader {
         key_len: key.serialized_len() as u32,
         value_len: value.serialized_len() as u32,
+        sequence,
         checksum,
     };
     header.write(&mut buf[..EntryHeader::serialized_len()]);
@@ -889,7 +935,7 @@ where
         }))
     }
 
-    pub async fn next(&mut self) -> Result<Option<Index<K>>> {
+    pub async fn next(&mut self) -> Result<Option<(K, IndexInfo)>> {
         let region_size = self.region.device().region_size();
         let align = self.region.device().align();
 
@@ -947,31 +993,37 @@ where
             key
         };
 
-        let index = Index {
-            key,
-            region: self.region.id(),
-            version: 0,
-            offset: self.cursor as u32,
-            len: entry_len as u32,
-            key_len: header.key_len,
-            value_len: header.value_len,
+        let info = IndexInfo {
+            sequence: header.sequence,
+            index: Index::Region {
+                region: self.region.id(),
+                version: 0,
+                offset: self.cursor as u32,
+                len: entry_len as u32,
+                key_len: header.key_len,
+                value_len: header.value_len,
+            },
         };
 
         self.cursor += entry_len;
 
-        Ok(Some(index))
+        Ok(Some((key, info)))
     }
 
     pub async fn next_kv(&mut self) -> Result<Option<(K, V)>> {
-        let index = match self.next().await {
-            Ok(Some(index)) => index,
+        let (_, info) = match self.next().await {
+            Ok(Some(res)) => res,
             Ok(None) => return Ok(None),
             Err(e) => return Err(e),
         };
 
+        let Index::Region { offset, len, .. } = info.index else {
+            unreachable!("kv loaded from region must have index of region")
+        };
+
         // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
-        let start = index.offset as usize;
-        let end = start + index.len as usize;
+        let start = offset as usize;
+        let end = start + len as usize;
         let Some(slice) = self.region.load(start..end, 0).await? else {
             return Ok(None);
         };
@@ -1105,6 +1157,7 @@ mod tests {
                 io_size: 4 * KB,
             },
             allocator_bits: 1,
+            catalog_bits: 1,
             admissions,
             reinsertions,
             buffer_pool_size: 8 * MB,
@@ -1156,6 +1209,7 @@ mod tests {
                 io_size: 4096 * KB,
             },
             allocator_bits: 1,
+            catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index b680589f..555a60e2 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -239,6 +239,7 @@ mod tests {
                 io_size: 4096 * KB,
             },
             allocator_bits: 1,
+            catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
@@ -274,6 +275,7 @@ mod tests {
                 io_size: 4096 * KB,
             },
             allocator_bits: 1,
+            catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 2e05bcc1..9ca902b9 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -25,11 +25,11 @@
 #![feature(associated_type_defaults)]
 
 pub mod admission;
+pub mod catalog;
 pub mod device;
 pub mod error;
 pub mod flusher;
 pub mod generic;
-pub mod indices;
 pub mod judge;
 pub mod lazy;
 pub mod metrics;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index f6f70f8b..d75f6fd5 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -113,7 +113,7 @@ where
         let region = self.region_manager.region(&region_id);
 
         // step 1: drop indices
-        let _indices = self.store.indices().take_region(&region_id);
+        let _indices = self.store.catalog().take_region(&region_id);
 
         // after drop indices and acquire exclusive lock, no writers or readers are supposed to access the region
         {
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index 95500032..39bf6d6a 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -19,7 +19,7 @@ use std::{
 
 use foyer_common::code::{Key, Value};
 
-use crate::indices::Indices;
+use crate::catalog::Catalog;
 
 use super::ReinsertionPolicy;
 
@@ -29,7 +29,7 @@ where
     K: Key,
     V: Value,
 {
-    indices: OnceLock<Arc<Indices<K>>>,
+    indices: OnceLock<Arc<Catalog<K>>>,
     _marker: PhantomData<V>,
 }
 
@@ -55,7 +55,7 @@ where
 
     type Value = V;
 
-    fn init(&self, indices: &Arc<crate::indices::Indices<Self::Key>>) {
+    fn init(&self, indices: &Arc<Catalog<Self::Key>>) {
         self.indices.get_or_init(|| indices.clone());
     }
 
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index bf3881de..fb383c56 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -14,7 +14,7 @@
 
 use foyer_common::code::{Key, Value};
 
-use crate::{indices::Indices, metrics::Metrics};
+use crate::{catalog::Catalog, metrics::Metrics};
 use std::{fmt::Debug, sync::Arc};
 
 #[expect(unused_variables)]
@@ -22,7 +22,7 @@ pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, indices: &Arc<Indices<Self::Key>>) {}
+    fn init(&self, indices: &Arc<Catalog<Self::Key>>) {}
 
     fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
 
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
new file mode 100644
index 00000000..f734a4b5
--- /dev/null
+++ b/foyer-storage/src/ring.rs
@@ -0,0 +1,223 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    alloc::Global,
+    collections::BTreeSet,
+    marker::PhantomData,
+    ops::{Deref, DerefMut},
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
+
+use foyer_common::code::{Key, Value};
+use parking_lot::Mutex;
+use tokio::sync::Notify;
+
+use crate::device::BufferAllocator;
+
+pub struct Buffer<K, V, A = Global>
+where
+    K: Key,
+    V: Value,
+    A: BufferAllocator,
+{
+    queue: Arc<Mutex<BTreeSet<SequenceOrderedItem<K, A>>>>,
+    index: Arc<Mutex<BTreeSet<KeyOrderedItem<K, A>>>>,
+
+    ring: Arc<RingBuffer<A>>,
+
+    _marker: PhantomData<V>,
+}
+
+impl<K, V, A> Buffer<K, V, A>
+where
+    K: Key,
+    V: Value,
+    A: BufferAllocator,
+{
+    pub async fn insert(&self, key: K, value: V) {
+        let mut view = loop {
+            if let Some(view) = self.ring.allocate(value.serialized_len()).await {
+                break view;
+            }
+        };
+
+        value.write(&mut view);
+
+        todo!()
+    }
+}
+
+struct Item<K, A = Global>
+where
+    K: Key,
+    A: BufferAllocator,
+{
+    sequence: usize,
+    key: K,
+    view: ViewMut<A>,
+}
+
+struct SequenceOrderedItem<K, A = Global>
+where
+    K: Key,
+    A: BufferAllocator,
+{
+    item: Arc<Item<K, A>>,
+}
+
+struct KeyOrderedItem<K, A = Global>
+where
+    K: Key,
+    A: BufferAllocator,
+{
+    item: Arc<Item<K, A>>,
+}
+
+#[derive(Debug)]
+pub struct RingBuffer<A = Global>
+where
+    A: BufferAllocator,
+{
+    data: Vec<u8, A>,
+    capacity: usize,
+
+    allocated: AtomicUsize,
+    released: AtomicUsize,
+
+    nofify: Notify,
+}
+
+impl<A> RingBuffer<A>
+where
+    A: BufferAllocator,
+{
+    /// Allocate a mutable view with required size on the ring buffer.
+    ///
+    /// Returns `None` when the allocated buffer cross the boundary.
+    pub async fn allocate(self: &Arc<Self>, len: usize) -> Option<ViewMut<A>> {
+        let offset = self.allocated.fetch_add(len, Ordering::SeqCst);
+
+        if offset / self.capacity != (offset + len) / self.capacity {
+            return None;
+        }
+        let offset = offset % self.capacity;
+
+        let view = Some(ViewMut::new(self, offset, len));
+
+        loop {
+            if self.released.load(Ordering::Acquire) >= offset + len {
+                break view;
+            }
+            self.nofify.notified().await;
+        }
+    }
+
+    /// Release buffer to `position`.
+    ///
+    /// The buffer between last `released` point and `position` will be released after
+    /// all views on it destroyed.
+    pub fn release(&self, position: usize) {
+        todo!()
+    }
+}
+
+/// # Safety
+///
+/// The underlying buffer of [`ViewMut`] must be valid during its lifetime.
+pub struct ViewMut<A = Global>
+where
+    A: BufferAllocator,
+{
+    ring: Arc<RingBuffer<A>>,
+    ptr: *mut u8,
+    len: usize,
+}
+
+impl<A> ViewMut<A>
+where
+    A: BufferAllocator,
+{
+    fn new(ring: &Arc<RingBuffer<A>>, offset: usize, len: usize) -> Self {
+        Self {
+            ring: Arc::clone(ring),
+            ptr: (ring.data.as_ptr() as usize + offset) as *mut u8,
+            len,
+        }
+    }
+}
+
+impl<A> Deref for ViewMut<A>
+where
+    A: BufferAllocator,
+{
+    type Target = [u8];
+
+    fn deref(&self) -> &Self::Target {
+        unsafe { core::slice::from_raw_parts(self.ptr, self.len) }
+    }
+}
+
+impl<A> DerefMut for ViewMut<A>
+where
+    A: BufferAllocator,
+{
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) }
+    }
+}
+
+unsafe impl<A> Send for ViewMut<A> where A: BufferAllocator {}
+unsafe impl<A> Sync for ViewMut<A> where A: BufferAllocator {}
+
+/// # Safety
+///
+/// The underlying buffer of [`View`] must be valid during its lifetime.
+pub struct View<A = Global>
+where
+    A: BufferAllocator,
+{
+    view: ViewMut<A>,
+}
+
+impl<A> View<A>
+where
+    A: BufferAllocator,
+{
+    fn new(ring: &Arc<RingBuffer<A>>, offset: usize, len: usize) -> Self {
+        let view = ViewMut {
+            ring: Arc::clone(ring),
+            ptr: (ring.data.as_ptr() as usize + offset) as *mut u8,
+            len,
+        };
+        Self { view }
+    }
+}
+
+impl<A> Deref for View<A>
+where
+    A: BufferAllocator,
+{
+    type Target = [u8];
+
+    fn deref(&self) -> &Self::Target {
+        self.view.deref()
+    }
+}
+
+unsafe impl<A> Send for View<A> where A: BufferAllocator {}
+unsafe impl<A> Sync for View<A> where A: BufferAllocator {}
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 3ab87242..51ac9668 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -129,6 +129,7 @@ async fn test_store() {
             io_size: 4 * KB,
         },
         allocator_bits: 0,
+        catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
         buffer_pool_size: 2 * MB,
@@ -159,6 +160,7 @@ async fn test_lazy_store() {
             io_size: 4 * KB,
         },
         allocator_bits: 0,
+        catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
         buffer_pool_size: 2 * MB,
@@ -190,6 +192,7 @@ async fn test_runtime_store() {
                 io_size: 4 * KB,
             },
             allocator_bits: 0,
+            catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
             buffer_pool_size: 2 * MB,
@@ -227,6 +230,7 @@ async fn test_runtime_lazy_store() {
                 io_size: 4 * KB,
             },
             allocator_bits: 0,
+            catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
             buffer_pool_size: 2 * MB,

From f7dcdef4fe71cb098222b42ff0f18ecb63e9c01d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 23 Oct 2023 01:09:31 -0500
Subject: [PATCH 147/261] chore: update codecov ci (#156)

* chore: update codecov ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* update

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix codecov.yml

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix codecov.yml

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* modify codecov.yml

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      | 4 +++-
 .github/workflows/main.yml         | 4 +++-
 .github/workflows/pull-request.yml | 4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index a9debcce..92cd47e6 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -81,7 +81,9 @@ jobs:
         run: |
           cargo llvm-cov --no-report nextest
           cargo llvm-cov report --lcov --output-path lcov.info
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v3
+        with:
+          verbose: true
   deadlock:
     name: run with single worker thread and deadlock detection
     runs-on: ubuntu-latest
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5aad8b4f..1b71f47c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -88,7 +88,9 @@ jobs:
         run: |
           cargo llvm-cov --no-report nextest
           cargo llvm-cov report --lcov --output-path lcov.info
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v3
+        with:
+          verbose: true
   deadlock:
     name: run with single worker thread and deadlock detection
     runs-on: ubuntu-latest
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index d5e50f1d..9a500331 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -87,7 +87,9 @@ jobs:
         run: |
           cargo llvm-cov --no-report nextest
           cargo llvm-cov report --lcov --output-path lcov.info
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v3
+        with:
+          verbose: true
   deadlock:
     name: run with single worker thread and deadlock detection
     runs-on: ubuntu-latest

From 571e4025e54bab8a783f6014adddeb9c681a0d65 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 23 Oct 2023 01:21:23 -0500
Subject: [PATCH 148/261] feat: introduce concurrent Continuum tracker (#181)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml       |   1 +
 foyer-common/src/continuum.rs | 228 ++++++++++++++++++++++++++++++++++
 foyer-common/src/lib.rs       |   1 +
 3 files changed, 230 insertions(+)
 create mode 100644 foyer-common/src/continuum.rs

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 93357a95..638de565 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -13,6 +13,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 bytes = "1"
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
+itertools = "0.11"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 rand = "0.8.5"
diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
new file mode 100644
index 00000000..b19eee4b
--- /dev/null
+++ b/foyer-common/src/continuum.rs
@@ -0,0 +1,228 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use itertools::Itertools;
+use std::{
+    ops::Range,
+    sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering},
+};
+
+macro_rules! def_continuum {
+    ($( { $name:ident, $uint:ty, $atomic:ty }, )*) => {
+        $(
+            #[derive(Debug)]
+            pub struct $name {
+                slots: Vec<$atomic>,
+                capacity: $uint,
+                continuum: $atomic,
+            }
+
+            impl $name {
+                /// `capacity` must be power of 2.
+                pub fn new(capacity: $uint) -> Self {
+                    assert!(capacity.is_power_of_two());
+                    let slots = (0..capacity).map(|_| <$atomic>::default()).collect_vec();
+                    let continuum = <$atomic>::default();
+                    Self {
+                        slots,
+                        capacity,
+                        continuum,
+                    }
+                }
+
+                pub fn submit(&self, range: Range<$uint>) {
+                    debug_assert!(range.start < range.end);
+
+                    let continuum = self.continuum.load(Ordering::Acquire);
+
+                    debug_assert!(continuum <= range.start, "assert continuum <= range.start failed: {} <= {}", continuum, range.start);
+
+                    if continuum == range.start {
+                        // continuum can be advanced directly and exclusively
+                        self.continuum.store(range.end, Ordering::Release);
+                    } else {
+                        let slot = &self.slots[self.slot(range.start)];
+                        slot.store(range.end, Ordering::Release);
+                        let stop = move |current: $uint, _next: $uint| {
+                            current > range.start
+                        };
+                        self.advance_until(stop, 1);
+                    }
+                }
+
+                pub fn advance(&self) -> bool {
+                    self.advance_until(|_, _| false, 0)
+                }
+
+                pub fn continuum(&self) -> $uint {
+                    self.continuum.load(Ordering::Acquire)
+                }
+
+                fn slot(&self, position: $uint) -> usize {
+                    (position & (self.capacity - 1)) as usize
+                }
+
+                /// `stop: Fn(continuum, next) -> bool`.
+                fn advance_until<P>(&self, stop: P, retry: usize) -> bool
+                where
+                    P: Fn($uint, $uint) -> bool + Send + Sync + 'static,
+                {
+                    let mut continuum = self.continuum.load(Ordering::Acquire);
+                    let mut start = continuum;
+
+                    let mut times = 0;
+                    loop {
+                        let slot = &self.slots[self.slot(continuum)];
+
+                        let next = slot.load(Ordering::Acquire);
+
+                        if next >= continuum + self.capacity {
+                            // case 1: `range` >= `capacity`
+                            // case 2: continuum has rotated before `next` is loaded
+                            continuum = self.continuum.load(Ordering::Acquire);
+                            if continuum != start {
+                                start = continuum;
+                                continue;
+                            }
+                        }
+
+                        if next <= continuum || stop(continuum, next) {
+                            // nothing to advance
+                            return false;
+                        }
+
+                        // make sure `continuum` can be modified exclusively and lock
+                        if let Ok(_) = slot.compare_exchange(next, continuum, Ordering::AcqRel, Ordering::Relaxed) {
+                            // If this thread is scheduled for a long time after `continuum` is loaded,
+                            // the `slot` may refer to a rotated slot with actual index `continuum + N * capacity`,
+                            // and the loaded `continuum` lags. `continuum` needs to be checked if it is still behind
+                            // the slot.
+                            continuum = self.continuum.load(Ordering::Acquire);
+                            if continuum == start {
+                                // exclusive
+                                continuum = next;
+                                break;
+                            }
+                        }
+
+                        // prepare for the next retry
+                        times += 1;
+                        if times > retry {
+                            return false;
+                        }
+
+                        continuum = self.continuum.load(Ordering::Acquire);
+                        if continuum == start {
+                            return false;
+                        }
+                        start = continuum;
+                    }
+
+                    loop {
+                        let next = self.slots[self.slot(continuum)].load(Ordering::Relaxed);
+
+                        if next <= continuum || stop(continuum, next) {
+                            break;
+                        }
+
+                        continuum = next;
+                    }
+
+                    #[cfg(test)]
+                    assert_eq!(start, self.continuum.load(Ordering::Acquire));
+
+                    // modify continuum exclusively and unlock
+                    self.continuum.store(continuum, Ordering::Release);
+
+                    if continuum == start {
+                        return false;
+                    }
+                    return true;
+                }
+            }
+        )*
+    }
+}
+
+macro_rules! for_all_primitives {
+    ($macro:ident) => {
+        $macro! {
+            { ContinuumU8, u8, AtomicU8 },
+            { ContinuumU16, u16, AtomicU16 },
+            { ContinuumU32, u32, AtomicU32 },
+            { ContinuumU64, u64, AtomicU64 },
+            { ContinuumUsize, usize, AtomicUsize },
+        }
+    };
+}
+
+for_all_primitives! { def_continuum }
+
+#[cfg(test)]
+mod tests {
+    use std::{sync::Arc, time::Duration};
+
+    use rand::{rngs::OsRng, Rng};
+    use tokio::sync::Semaphore;
+
+    use super::*;
+
+    #[ignore]
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_continuum_fuzzy() {
+        const CAPACITY: u64 = 4096;
+        const CURRENCY: usize = 16;
+        const UNIT: u64 = 16;
+        const LOOP: usize = 1000;
+
+        let s = Arc::new(Semaphore::new(CAPACITY as usize));
+        let c = Arc::new(ContinuumU64::new(CAPACITY));
+        let v = Arc::new(AtomicU64::new(0));
+
+        let tasks = (0..CURRENCY)
+            .map(|_| {
+                let s = s.clone();
+                let c = c.clone();
+                let v = v.clone();
+                async move {
+                    for _ in 0..LOOP {
+                        let unit = OsRng.gen_range(1..UNIT);
+                        let start = v.fetch_add(unit, Ordering::Relaxed);
+                        let end = start + unit;
+
+                        let permit = s.acquire_many(unit as u32).await.unwrap();
+
+                        let sleep = OsRng.gen_range(0..10);
+                        tokio::time::sleep(Duration::from_millis(sleep)).await;
+                        c.submit(start..end);
+
+                        drop(permit);
+                    }
+                }
+            })
+            .collect_vec();
+
+        let handles = tasks
+            .into_iter()
+            .map(|task| tokio::spawn(task))
+            .collect_vec();
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        c.advance();
+
+        assert_eq!(v.load(Ordering::Relaxed), c.continuum());
+    }
+}
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 0b6b776f..8e6e801e 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -20,6 +20,7 @@
 pub mod batch;
 pub mod bits;
 pub mod code;
+pub mod continuum;
 pub mod erwlock;
 pub mod queue;
 pub mod range;

From 5d0134b28c0edb03277b01ce08b035ef52c1b783 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 23 Oct 2023 11:17:14 -0500
Subject: [PATCH 149/261] feat: introduce ring buffer (#182)

* feat: introduce ring buffer

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      |   4 +-
 .github/workflows/main.yml         |   4 +-
 .github/workflows/pull-request.yml |   4 +-
 Makefile                           |   6 +-
 foyer-common/src/continuum.rs      |  30 ++-
 foyer-storage/src/lib.rs           |   1 +
 foyer-storage/src/ring.rs          | 334 +++++++++++++++++++++--------
 7 files changed, 286 insertions(+), 97 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 92cd47e6..5d74ac7d 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -76,10 +76,12 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-common -- --nocapture --ignored
+          cargo llvm-cov --no-report nextest --run-ignored ignored-only --no-capture --workspace
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov --no-report nextest
+      - name: Generate codecov report
+        run: |
           cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v3
         with:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 1b71f47c..6c1a5f80 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -83,10 +83,12 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-common -- --nocapture --ignored
+          cargo llvm-cov --no-report nextest --run-ignored ignored-only --no-capture --workspace
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov --no-report nextest
+      - name: Generate codecov report
+        run: |
           cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v3
         with:
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 9a500331..bfb4e59f 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -82,10 +82,12 @@ jobs:
         uses: taiki-e/install-action@nextest
       - name: Run rust test with coverage (igored tests)
         run: |
-          cargo llvm-cov --no-report test --package foyer-common -- --nocapture --ignored
+          cargo llvm-cov --no-report nextest --run-ignored ignored-only --no-capture --workspace
       - name: Run rust test with coverage
         run: |
           cargo llvm-cov --no-report nextest
+      - name: Generate codecov report
+        run: |
           cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v3
         with:
diff --git a/Makefile b/Makefile
index ddc7728a..4fea9a07 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: deps check test test-ignored test-all all monitor clear madsim
+.PHONY: deps check test test-ignored test-all all fast monitor clear madsim
 
 deps:
 	./scripts/install-deps.sh
@@ -21,7 +21,7 @@ test:
 	RUST_BACKTRACE=1 cargo test --doc
 
 test-ignored:
-	RUST_BACKTRACE=1 cargo test --package foyer-common -- --nocapture --ignored
+	RUST_BACKTRACE=1 cargo nextest run --run-ignored ignored-only --no-capture --workspace
 
 test-all: test test-ignored
 
@@ -32,6 +32,8 @@ madsim:
 
 all: check test-all
 
+fast: check test
+
 monitor:
 	./scripts/monitor.sh
 
diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
index b19eee4b..3106395f 100644
--- a/foyer-common/src/continuum.rs
+++ b/foyer-common/src/continuum.rs
@@ -41,9 +41,35 @@ macro_rules! def_continuum {
                     }
                 }
 
+                pub fn is_occupied(&self, start: $uint) -> bool {
+                    !self.is_vacant(start)
+                }
+
+                pub fn is_vacant(&self, start: $uint) -> bool {
+                    let continuum = self.continuum.load(Ordering::Acquire);
+                    if continuum + self.capacity > start {
+                        return true;
+                    }
+
+                    self.advance_until(|_, _| false, 0);
+
+                    let continuum = self.continuum.load(Ordering::Acquire);
+                    continuum + self.capacity > start
+                }
+
+                /// Submit a range.
                 pub fn submit(&self, range: Range<$uint>) {
                     debug_assert!(range.start < range.end);
 
+                    self.slots[self.slot(range.start)].store(range.end, Ordering::SeqCst);
+                }
+
+                /// Submit a range, may advance continuum till the given range.
+                ///
+                /// Return `true` if advanced, else `false`.
+                pub fn submit_advance(&self, range: Range<$uint>) -> bool{
+                    debug_assert!(range.start < range.end);
+
                     let continuum = self.continuum.load(Ordering::Acquire);
 
                     debug_assert!(continuum <= range.start, "assert continuum <= range.start failed: {} <= {}", continuum, range.start);
@@ -51,13 +77,14 @@ macro_rules! def_continuum {
                     if continuum == range.start {
                         // continuum can be advanced directly and exclusively
                         self.continuum.store(range.end, Ordering::Release);
+                        true
                     } else {
                         let slot = &self.slots[self.slot(range.start)];
                         slot.store(range.end, Ordering::Release);
                         let stop = move |current: $uint, _next: $uint| {
                             current > range.start
                         };
-                        self.advance_until(stop, 1);
+                        self.advance_until(stop, 1)
                     }
                 }
 
@@ -206,6 +233,7 @@ mod tests {
                         let sleep = OsRng.gen_range(0..10);
                         tokio::time::sleep(Duration::from_millis(sleep)).await;
                         c.submit(start..end);
+                        c.advance();
 
                         drop(permit);
                     }
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 9ca902b9..98cbdaf1 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -37,6 +37,7 @@ pub mod reclaimer;
 pub mod region;
 pub mod region_manager;
 pub mod reinsertion;
+pub mod ring;
 pub mod runtime;
 pub mod slice;
 pub mod storage;
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index f734a4b5..d3369744 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -12,153 +12,163 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use crate::{catalog::Sequence, device::BufferAllocator};
+use foyer_common::{bits::align_up, continuum::ContinuumUsize};
+use itertools::Itertools;
 use std::{
     alloc::Global,
-    collections::BTreeSet,
-    marker::PhantomData,
     ops::{Deref, DerefMut},
     sync::{
         atomic::{AtomicUsize, Ordering},
         Arc,
     },
+    time::Duration,
 };
 
-use foyer_common::code::{Key, Value};
-use parking_lot::Mutex;
-use tokio::sync::Notify;
-
-use crate::device::BufferAllocator;
-
-pub struct Buffer<K, V, A = Global>
+#[derive(Debug)]
+pub struct RingBuffer<A = Global>
 where
-    K: Key,
-    V: Value,
     A: BufferAllocator,
 {
-    queue: Arc<Mutex<BTreeSet<SequenceOrderedItem<K, A>>>>,
-    index: Arc<Mutex<BTreeSet<KeyOrderedItem<K, A>>>>,
-
-    ring: Arc<RingBuffer<A>>,
+    data: Vec<u8, A>,
+    align: usize,
+    capacity: usize,
+    blocks: usize,
 
-    _marker: PhantomData<V>,
-}
+    allocated: AtomicUsize,
 
-impl<K, V, A> Buffer<K, V, A>
-where
-    K: Key,
-    V: Value,
-    A: BufferAllocator,
-{
-    pub async fn insert(&self, key: K, value: V) {
-        let mut view = loop {
-            if let Some(view) = self.ring.allocate(value.serialized_len()).await {
-                break view;
-            }
-        };
+    refs: Vec<Arc<AtomicUsize>>,
 
-        value.write(&mut view);
+    continuum: Arc<ContinuumUsize>,
+}
 
-        todo!()
+impl RingBuffer<Global> {
+    /// `align` must be power of 2.
+    ///
+    /// `capacity` must be a multiplier of `align`.
+    pub fn new(align: usize, blocks: usize) -> Self {
+        Self::new_in(align, blocks, Global)
     }
 }
 
-struct Item<K, A = Global>
+impl<A> RingBuffer<A>
 where
-    K: Key,
     A: BufferAllocator,
 {
-    sequence: usize,
-    key: K,
-    view: ViewMut<A>,
-}
+    /// `align` must be power of 2.
+    ///
+    /// `blocks` must be power of 2.
+    ///
+    /// `capacity = align * blocks`.
+    pub fn new_in(align: usize, blocks: usize, alloc: A) -> Self {
+        assert!(align.is_power_of_two());
+        assert!(blocks.is_power_of_two());
 
-struct SequenceOrderedItem<K, A = Global>
-where
-    K: Key,
-    A: BufferAllocator,
-{
-    item: Arc<Item<K, A>>,
-}
+        let capacity = align * blocks;
 
-struct KeyOrderedItem<K, A = Global>
-where
-    K: Key,
-    A: BufferAllocator,
-{
-    item: Arc<Item<K, A>>,
-}
+        let mut data = Vec::with_capacity_in(capacity, alloc);
+        unsafe { data.set_len(capacity) };
 
-#[derive(Debug)]
-pub struct RingBuffer<A = Global>
-where
-    A: BufferAllocator,
-{
-    data: Vec<u8, A>,
-    capacity: usize,
+        let allocated = AtomicUsize::new(0);
 
-    allocated: AtomicUsize,
-    released: AtomicUsize,
+        let blocks = capacity / align;
+        let continuum = Arc::new(ContinuumUsize::new(blocks));
+        let refs = (0..blocks)
+            .map(|_| Arc::new(AtomicUsize::default()))
+            .collect_vec();
 
-    nofify: Notify,
-}
+        Self {
+            data,
+            align,
+            capacity,
+            blocks,
+            allocated,
+            refs,
+            continuum,
+        }
+    }
 
-impl<A> RingBuffer<A>
-where
-    A: BufferAllocator,
-{
     /// Allocate a mutable view with required size on the ring buffer.
     ///
     /// Returns `None` when the allocated buffer cross the boundary.
-    pub async fn allocate(self: &Arc<Self>, len: usize) -> Option<ViewMut<A>> {
-        let offset = self.allocated.fetch_add(len, Ordering::SeqCst);
-
-        if offset / self.capacity != (offset + len) / self.capacity {
-            return None;
+    ///
+    /// When all views from an allocation are dropped, the buffer will be released.
+    pub async fn allocate(self: &Arc<Self>, len: usize, sequence: Sequence) -> ViewMut<A> {
+        loop {
+            if let Some(view) = self.allocate_inner(len, sequence).await {
+                return view;
+            }
         }
-        let offset = offset % self.capacity;
+    }
+
+    async fn allocate_inner(
+        self: &Arc<Self>,
+        len: usize,
+        sequence: Sequence,
+    ) -> Option<ViewMut<A>> {
+        let len = align_up(self.align, len);
+        let offset = self.allocated.fetch_add(len, Ordering::SeqCst);
 
-        let view = Some(ViewMut::new(self, offset, len));
+        debug_assert_eq!(offset & (self.align - 1), 0);
 
         loop {
-            if self.released.load(Ordering::Acquire) >= offset + len {
-                break view;
+            self.continuum.advance();
+            if self.continuum.continuum() * self.align + self.capacity >= offset + len {
+                break;
             }
-            self.nofify.notified().await;
+            tokio::time::sleep(Duration::from_micros(100)).await;
         }
+
+        if offset / self.capacity != (offset + len) / self.capacity {
+            debug_assert!(self.continuum.is_vacant(offset / self.align));
+
+            self.continuum
+                .submit((offset / self.align)..((offset + len) / self.align));
+            return None;
+        }
+
+        let refs = self.refs(sequence);
+        Some(ViewMut::new(self, offset, len, refs))
     }
 
-    /// Release buffer to `position`.
-    ///
-    /// The buffer between last `released` point and `position` will be released after
-    /// all views on it destroyed.
-    pub fn release(&self, position: usize) {
-        todo!()
+    fn refs(&self, sequence: Sequence) -> &Arc<AtomicUsize> {
+        &self.refs[sequence as usize & (self.blocks - 1)]
     }
 }
 
 /// # Safety
 ///
 /// The underlying buffer of [`ViewMut`] must be valid during its lifetime.
+#[derive(Debug)]
 pub struct ViewMut<A = Global>
 where
     A: BufferAllocator,
 {
     ring: Arc<RingBuffer<A>>,
     ptr: *mut u8,
+    offset: usize,
     len: usize,
+    refs: Arc<AtomicUsize>,
 }
 
 impl<A> ViewMut<A>
 where
     A: BufferAllocator,
 {
-    fn new(ring: &Arc<RingBuffer<A>>, offset: usize, len: usize) -> Self {
+    fn new(ring: &Arc<RingBuffer<A>>, offset: usize, len: usize, refs: &Arc<AtomicUsize>) -> Self {
+        refs.fetch_add(1, Ordering::AcqRel);
         Self {
             ring: Arc::clone(ring),
-            ptr: (ring.data.as_ptr() as usize + offset) as *mut u8,
+            ptr: (ring.data.as_ptr() as usize + (offset & (ring.capacity - 1))) as *mut u8,
+            offset,
             len,
+            refs: Arc::clone(refs),
         }
     }
+
+    pub fn freeze(self) -> View<A> {
+        View::from(self)
+    }
 }
 
 impl<A> Deref for ViewMut<A>
@@ -181,12 +191,27 @@ where
     }
 }
 
+impl<A> Drop for ViewMut<A>
+where
+    A: BufferAllocator,
+{
+    fn drop(&mut self) {
+        if self.refs.fetch_sub(1, Ordering::AcqRel) == 1 {
+            let block_start = self.offset / self.ring.align;
+            let block_end = (self.offset + self.len) / self.ring.align;
+            self.ring.continuum.is_vacant(block_start);
+            self.ring.continuum.submit(block_start..block_end);
+        }
+    }
+}
+
 unsafe impl<A> Send for ViewMut<A> where A: BufferAllocator {}
 unsafe impl<A> Sync for ViewMut<A> where A: BufferAllocator {}
 
 /// # Safety
 ///
 /// The underlying buffer of [`View`] must be valid during its lifetime.
+#[derive(Debug)]
 pub struct View<A = Global>
 where
     A: BufferAllocator,
@@ -194,16 +219,11 @@ where
     view: ViewMut<A>,
 }
 
-impl<A> View<A>
+impl<A> From<ViewMut<A>> for View<A>
 where
     A: BufferAllocator,
 {
-    fn new(ring: &Arc<RingBuffer<A>>, offset: usize, len: usize) -> Self {
-        let view = ViewMut {
-            ring: Arc::clone(ring),
-            ptr: (ring.data.as_ptr() as usize + offset) as *mut u8,
-            len,
-        };
+    fn from(view: ViewMut<A>) -> Self {
         Self { view }
     }
 }
@@ -219,5 +239,137 @@ where
     }
 }
 
+impl<A> Clone for View<A>
+where
+    A: BufferAllocator,
+{
+    fn clone(&self) -> Self {
+        self.view.refs.fetch_add(1, Ordering::AcqRel);
+        let view = ViewMut {
+            ring: self.view.ring.clone(),
+            ptr: self.view.ptr,
+            offset: self.view.offset,
+            len: self.view.len,
+            refs: self.view.refs.clone(),
+        };
+        Self { view }
+    }
+}
+
 unsafe impl<A> Send for View<A> where A: BufferAllocator {}
 unsafe impl<A> Sync for View<A> where A: BufferAllocator {}
+
+#[cfg(test)]
+mod tests {
+
+    use std::{
+        collections::BTreeMap,
+        future::{poll_fn, Future},
+        ops::Range,
+        pin::pin,
+        sync::atomic::AtomicU64,
+        task::{Poll, Poll::Pending},
+        thread::available_parallelism,
+        time::Instant,
+    };
+
+    use bytes::BufMut;
+    use rand::{rngs::OsRng, Rng};
+
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_ring() {
+        const ALIGN: usize = 4096; // 4 KiB
+        const BLOCKS: usize = 4096; // capacity = 4 KiB * 4K = 16 MiB
+
+        let ring = Arc::new(RingBuffer::new(ALIGN, BLOCKS));
+        let sequence = Arc::new(AtomicU64::default());
+
+        let mut views = BTreeMap::new();
+        for i in 0..15 {
+            let seq = sequence.fetch_add(1, Ordering::Relaxed);
+            let view = ring
+                .allocate_inner(1024 * 1024 - 3000 /* ~ 1 MiB */, seq)
+                .await
+                .unwrap();
+            assert_eq!(view.offset, i * 1024 * 1024);
+            assert_eq!(view.len, 1024 * 1024);
+            views.insert(seq, view);
+        }
+        let seq = sequence.fetch_add(1, Ordering::Relaxed);
+
+        let mut future = pin!(ring.allocate_inner(2 * 1024 * 1024 - 3000 /* ~ 2 MiB */, seq));
+        assert!(matches! { poll_fn(|cx| Poll::Ready(future.as_mut().poll(cx))).await, Pending });
+        views.remove(&0).unwrap();
+        let res = future.await;
+        assert!(res.is_none());
+
+        let mut future = pin!(ring.allocate_inner(2 * 1024 * 1024 - 3000 /* ~ 2 MiB */, seq));
+        assert!(matches! { poll_fn(|cx| Poll::Ready(future.as_mut().poll(cx))).await, Pending });
+        views.remove(&2).unwrap();
+        views.remove(&1).unwrap();
+        let view = future.await.unwrap();
+        assert_eq!(view.offset, 17 * 1024 * 1024);
+        assert_eq!(view.len, 2 * 1024 * 1024);
+        views.insert(seq, view);
+
+        drop(views);
+    }
+
+    async fn test_ring_concurrent_case(blocks: usize, concurrency: usize, loops: usize) {
+        const ALIGN: usize = 4096; // 4 KiB
+        const SIZE: Range<usize> = 16 * 1024..256 * 1024; // 16 KiB ~ 128 KiB
+
+        let ring = Arc::new(RingBuffer::new(ALIGN, blocks));
+        let sequence = Arc::new(AtomicU64::default());
+
+        let tasks = (0..concurrency)
+            .map(|_| {
+                let ring = ring.clone();
+                let sequence = sequence.clone();
+                async move {
+                    for i in 0..loops {
+                        let seq = sequence.fetch_add(1, Ordering::Relaxed);
+                        let size = OsRng.gen_range(SIZE);
+                        let mut view = ring.allocate(size, seq).await;
+                        tokio::time::sleep(Duration::from_millis(OsRng.gen_range(0..10))).await;
+                        let data = vec![i as u8; size];
+                        view.as_mut().put_slice(&data);
+                        let view = view.freeze();
+                        assert!(view[..size] == data);
+                        drop(view);
+                    }
+                }
+            })
+            .collect_vec();
+        let handles = tasks
+            .into_iter()
+            .map(|task| tokio::spawn(task))
+            .collect_vec();
+        for handle in handles {
+            handle.await.unwrap();
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_ring_concurrent_small() {
+        // 4096 * 4096 = 16 MiB
+        test_ring_concurrent_case(4096, 16, 100).await;
+    }
+
+    #[ignore]
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_ring_concurrent_large() {
+        let concurrency = available_parallelism().unwrap().get() * 64;
+
+        let now = Instant::now();
+        test_ring_concurrent_case(65536, available_parallelism().unwrap().get() * 64, 1000).await;
+        let elapsed = now.elapsed();
+
+        println!("========== ring current fuzzy begin ==========");
+        println!("concurrency: {concurrency}");
+        println!("elapsed: {elapsed:?}");
+        println!("=========== ring current fuzzy end ===========");
+    }
+}

From 2261151107ad362851f5fff9ce4fa56e61911b10 Mon Sep 17 00:00:00 2001
From: TennyZhuang <zty0826@gmail.com>
Date: Tue, 24 Oct 2023 17:38:55 +0800
Subject: [PATCH 150/261] chore: bump toolchain to 2023-10-21 (#183)

* fix clippy

Signed-off-by: TennyZhuang <zty0826@gmail.com>

* update version in action

Signed-off-by: TennyZhuang <zty0826@gmail.com>

---------

Signed-off-by: TennyZhuang <zty0826@gmail.com>
---
 .github/template/template.yml      |  4 ++--
 .github/workflows/main.yml         |  2 +-
 .github/workflows/pull-request.yml |  2 +-
 foyer-common/src/code.rs           | 36 +++++++++++++++---------------
 foyer-common/src/continuum.rs      |  5 +----
 foyer-common/src/lib.rs            |  2 +-
 foyer-intrusive/src/lib.rs         |  1 -
 foyer-storage-bench/src/main.rs    |  1 -
 foyer-storage/src/lib.rs           |  2 --
 foyer-storage/src/ring.rs          |  5 +----
 foyer-storage/src/store.rs         |  1 -
 rust-toolchain                     |  2 +-
 12 files changed, 26 insertions(+), 37 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 5d74ac7d..79ba64ee 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -3,7 +3,7 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-09-09
+  RUST_TOOLCHAIN: nightly-2023-10-21
   CARGO_TERM_COLOR: always
   CACHE_KEY_SUFFIX: 20231010v3
 
@@ -183,4 +183,4 @@ jobs:
           RUST_LOG: info
           TOKIO_WORKER_THREADS: 1
         run: |-
-          cargo nextest run --all
\ No newline at end of file
+          cargo nextest run --all
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 6c1a5f80..581e1290 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,7 +11,7 @@ on:
     branches: [main]
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2023-09-09
+  RUST_TOOLCHAIN: nightly-2023-10-21
   CARGO_TERM_COLOR: always
   CACHE_KEY_SUFFIX: 20231010v3
 jobs:
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index bfb4e59f..553a573b 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -10,7 +10,7 @@ on:
   pull_request:
     branches: [main]
 env:
-  RUST_TOOLCHAIN: nightly-2023-09-09
+  RUST_TOOLCHAIN: nightly-2023-10-21
   CARGO_TERM_COLOR: always
   CACHE_KEY_SUFFIX: 20231010v3
 jobs:
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index d73dfa41..275a315f 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -29,22 +29,22 @@ pub trait Key:
     + Clone
     + std::fmt::Debug
 {
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Key` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
@@ -52,22 +52,22 @@ pub trait Key:
 
 #[expect(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Value` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
@@ -87,17 +87,17 @@ macro_rules! impl_key {
         paste! {
             $(
                 impl Key for $type {
-                    #[cfg_attr(coverage_nightly, no_coverage)]
+                    #[cfg_attr(coverage_nightly, coverage(off))]
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
                     }
 
-                    #[cfg_attr(coverage_nightly, no_coverage)]
+                    #[cfg_attr(coverage_nightly, coverage(off))]
                     fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
-                    #[cfg_attr(coverage_nightly, no_coverage)]
+                    #[cfg_attr(coverage_nightly, coverage(off))]
                     fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
@@ -112,17 +112,17 @@ macro_rules! impl_value {
         paste! {
             $(
                 impl Value for $type {
-                    #[cfg_attr(coverage_nightly, no_coverage)]
+                    #[cfg_attr(coverage_nightly, coverage(off))]
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
                     }
 
-                    #[cfg_attr(coverage_nightly, no_coverage)]
+                    #[cfg_attr(coverage_nightly, coverage(off))]
                     fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
-                    #[cfg_attr(coverage_nightly, no_coverage)]
+                    #[cfg_attr(coverage_nightly, coverage(off))]
                     fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
@@ -136,22 +136,22 @@ for_all_primitives! { impl_key }
 for_all_primitives! { impl_value }
 
 impl Value for Vec<u8> {
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn weight(&self) -> usize {
         self.len()
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn serialized_len(&self) -> usize {
         self.len()
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn write(&self, mut buf: &mut [u8]) {
         buf.put_slice(self);
     }
 
-    #[cfg_attr(coverage_nightly, no_coverage)]
+    #[cfg_attr(coverage_nightly, coverage(off))]
     fn read(buf: &[u8]) -> Self {
         buf.to_vec()
     }
diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
index 3106395f..2312ec5c 100644
--- a/foyer-common/src/continuum.rs
+++ b/foyer-common/src/continuum.rs
@@ -241,10 +241,7 @@ mod tests {
             })
             .collect_vec();
 
-        let handles = tasks
-            .into_iter()
-            .map(|task| tokio::spawn(task))
-            .collect_vec();
+        let handles = tasks.into_iter().map(tokio::spawn).collect_vec();
         for handle in handles {
             handle.await.unwrap();
         }
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 8e6e801e..e66098dc 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -15,7 +15,7 @@
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
 #![feature(bound_map)]
-#![cfg_attr(coverage_nightly, feature(no_coverage))]
+#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 
 pub mod batch;
 pub mod bits;
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index ce6b0d32..8f93d2fd 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -15,7 +15,6 @@
 #![feature(associated_type_bounds)]
 #![feature(ptr_metadata)]
 #![feature(trait_alias)]
-#![feature(return_position_impl_trait_in_trait)]
 #![feature(lint_reasons)]
 #![expect(clippy::new_without_default)]
 #![cfg_attr(test, expect(clippy::vtable_address_comparisons))]
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index ad8c4e91..0f02526f 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -14,7 +14,6 @@
 
 #![feature(let_chains)]
 #![feature(lint_reasons)]
-#![feature(async_fn_in_trait)]
 
 mod analyze;
 mod export;
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 98cbdaf1..49b31473 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -20,8 +20,6 @@
 #![feature(error_generic_member_access)]
 #![feature(lazy_cell)]
 #![feature(lint_reasons)]
-#![feature(async_fn_in_trait)]
-#![feature(return_position_impl_trait_in_trait)]
 #![feature(associated_type_defaults)]
 
 pub mod admission;
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index d3369744..0262609c 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -343,10 +343,7 @@ mod tests {
                 }
             })
             .collect_vec();
-        let handles = tasks
-            .into_iter()
-            .map(|task| tokio::spawn(task))
-            .collect_vec();
+        let handles = tasks.into_iter().map(tokio::spawn).collect_vec();
         for handle in handles {
             handle.await.unwrap();
         }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 6d694616..597cd553 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -117,7 +117,6 @@ impl<K: Key, V: Value> Storage for NoneStore<K, V> {
     type Config = ();
     type Writer = NoneStoreWriter<K, V>;
 
-    #[expect(clippy::let_unit_value)]
     async fn open(_: Self::Config) -> Result<Self> {
         Ok(NoneStore(PhantomData))
     }
diff --git a/rust-toolchain b/rust-toolchain
index 20d4da12..fe2a026f 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-09-09"
\ No newline at end of file
+channel = "nightly-2023-10-21"

From 5b3ff1861521b9c7130170249ffdc5d6e814a758 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 26 Oct 2023 11:54:29 +0800
Subject: [PATCH 151/261] chore: use semaphore to control recovery concurrency
 (#186)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/Cargo.toml        |  1 -
 foyer-storage/src/generic.rs    | 14 ++++++++------
 foyer-workspace-hack/Cargo.toml |  1 +
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 86f801b0..d99ceb29 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -12,7 +12,6 @@ normal = ["foyer-workspace-hack"]
 
 [dependencies]
 anyhow = "1.0"
-async-channel = "1.8"
 bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 7709d1b1..402a1195 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -29,7 +29,10 @@ use foyer_intrusive::eviction::EvictionPolicy;
 use futures::future::try_join_all;
 use itertools::Itertools;
 use parking_lot::Mutex;
-use tokio::{sync::broadcast, task::JoinHandle};
+use tokio::{
+    sync::{broadcast, Semaphore},
+    task::JoinHandle,
+};
 use twox_hash::XxHash64;
 
 use crate::{
@@ -492,18 +495,17 @@ where
     async fn recover(&self, concurrency: usize) -> Result<Sequence> {
         tracing::info!("start store recovery");
 
-        let (tx, rx) = async_channel::bounded(concurrency);
+        let semaphore = Arc::new(Semaphore::new(concurrency));
 
         let mut handles = vec![];
         for region_id in 0..self.inner.device.regions() as RegionId {
-            let itx = tx.clone();
-            let irx = rx.clone();
+            let semaphore = semaphore.clone();
             let region_manager = self.inner.region_manager.clone();
             let indices = self.inner.indices.clone();
             let handle = tokio::spawn(async move {
-                itx.send(()).await.unwrap();
+                let permit = semaphore.acquire().await;
                 let res = Self::recover_region(region_id, region_manager, indices).await;
-                irx.recv().await.unwrap();
+                drop(permit);
                 res
             });
             handles.push(handle);
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index d9926347..ab2c5188 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -16,6 +16,7 @@ publish = false
 crossbeam-utils = { version = "0.8" }
 either = { version = "1", default-features = false, features = ["use_std"] }
 futures-channel = { version = "0.3", features = ["sink"] }
+futures-core = { version = "0.3" }
 futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }

From 6ebd85cd65940f6f38de85474581c3223f283599 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 26 Oct 2023 17:48:01 +0800
Subject: [PATCH 152/261] chore: introduce node exporter full in Grafana (#187)

* chore: introduce node exporter full in Grafana

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add dashboard minimize script and ci check

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix scripts

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/generate.sh                  |     5 +-
 .github/template/template.yml                 |     7 +-
 .github/workflows/main.yml                    |     7 +-
 .github/workflows/pull-request.yml            |     7 +-
 Makefile                                      |     2 +
 etc/grafana/dashboards/foyer.min.json         |     1 +
 .../dashboards/node-exporter-full.json        | 23348 ++++++++++++++++
 .../dashboards/node-exporter-full.min.json    |     1 +
 etc/grafana/provisioning/dashboards/foyer.yml |     8 +-
 scripts/minimize-dashboards.sh                |    24 +
 10 files changed, 23404 insertions(+), 6 deletions(-)
 create mode 100644 etc/grafana/dashboards/foyer.min.json
 create mode 100644 etc/grafana/dashboards/node-exporter-full.json
 create mode 100644 etc/grafana/dashboards/node-exporter-full.min.json
 create mode 100755 scripts/minimize-dashboards.sh

diff --git a/.github/template/generate.sh b/.github/template/generate.sh
index 57a5b23e..2d7e27c4 100755
--- a/.github/template/generate.sh
+++ b/.github/template/generate.sh
@@ -1,12 +1,13 @@
 #!/bin/bash
 
+# You will need to install yq >= 4.16 to use this tool.
+# brew install yq
+
 set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR"
 
-# You will need to install yq >= 4.16 to use this tool.
-# brew install yq
 
 HEADER="""
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
diff --git a/.github/template/template.yml b/.github/template/template.yml
index 79ba64ee..93d163e1 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -14,16 +14,21 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install tools
+      - name: Install yq
         run: |
           wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
         env:
           YQ_VERSION: v4.16.1
           BINARY: yq_linux_amd64
           BUF_VERSION: 1.0.0-rc6
+      - name: Install jq
+        uses: dcarbone/install-jq-action@v2.0.2    
       - name: Check if CI workflows are up-to-date
         run: |
           ./.github/template/generate.sh --check
+      - name: Check if Grafana dashboards are minimized
+        run: |
+          ./scripts/minimize-dashboards.sh --check
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
   rust-test:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 581e1290..ea39db0c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -21,16 +21,21 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install tools
+      - name: Install yq
         run: |
           wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
         env:
           YQ_VERSION: v4.16.1
           BINARY: yq_linux_amd64
           BUF_VERSION: 1.0.0-rc6
+      - name: Install jq
+        uses: dcarbone/install-jq-action@v2.0.2
       - name: Check if CI workflows are up-to-date
         run: |
           ./.github/template/generate.sh --check
+      - name: Check if Grafana dashboards are minimized
+        run: |
+          ./scripts/minimize-dashboards.sh --check
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
   rust-test:
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 553a573b..2d88b355 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -20,16 +20,21 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install tools
+      - name: Install yq
         run: |
           wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
         env:
           YQ_VERSION: v4.16.1
           BINARY: yq_linux_amd64
           BUF_VERSION: 1.0.0-rc6
+      - name: Install jq
+        uses: dcarbone/install-jq-action@v2.0.2
       - name: Check if CI workflows are up-to-date
         run: |
           ./.github/template/generate.sh --check
+      - name: Check if Grafana dashboards are minimized
+        run: |
+          ./scripts/minimize-dashboards.sh --check
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
   rust-test:
diff --git a/Makefile b/Makefile
index 4fea9a07..117bea23 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,8 @@ deps:
 
 check:
 	shellcheck ./scripts/*
+	./.github/template/generate.sh
+	./scripts/minimize-dashboards.sh
 	cargo hakari generate
 	cargo hakari manage-deps
 	cargo sort -w
diff --git a/etc/grafana/dashboards/foyer.min.json b/etc/grafana/dashboards/foyer.min.json
new file mode 100644
index 00000000..50d5f8eb
--- /dev/null
+++ b/etc/grafana/dashboards/foyer.min.json
@@ -0,0 +1 @@
+{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":2,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":2,"weekStart":""}
diff --git a/etc/grafana/dashboards/node-exporter-full.json b/etc/grafana/dashboards/node-exporter-full.json
new file mode 100644
index 00000000..05527cb9
--- /dev/null
+++ b/etc/grafana/dashboards/node-exporter-full.json
@@ -0,0 +1,23348 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "$$hashKey": "object:1058",
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 1860,
+  "graphTooltip": 1,
+  "id": 2,
+  "links": [
+    {
+      "icon": "external link",
+      "tags": [],
+      "targetBlank": true,
+      "title": "GitHub",
+      "type": "link",
+      "url": "https://github.com/rfmoz/grafana-dashboards"
+    },
+    {
+      "icon": "external link",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Grafana",
+      "type": "link",
+      "url": "https://grafana.com/grafana/dashboards/1860"
+    }
+  ],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 261,
+      "panels": [],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Quick CPU / Mem / Disk",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Busy state of all CPU cores together",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgba(50, 172, 45, 0.97)",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 85
+              },
+              {
+                "color": "rgba(245, 54, 54, 0.9)",
+                "value": 95
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 0,
+        "y": 1
+      },
+      "id": 20,
+      "links": [],
+      "options": {
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100",
+          "hide": false,
+          "instant": true,
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "CPU Busy",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Busy state of all CPU cores together (5 min average)",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgba(50, 172, 45, 0.97)",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 85
+              },
+              {
+                "color": "rgba(245, 54, 54, 0.9)",
+                "value": 95
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 3,
+        "y": 1
+      },
+      "id": 155,
+      "links": [],
+      "options": {
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))",
+          "format": "time_series",
+          "hide": false,
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "Sys Load (5m avg)",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Busy state of all CPU cores together (15 min average)",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgba(50, 172, 45, 0.97)",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 85
+              },
+              {
+                "color": "rgba(245, 54, 54, 0.9)",
+                "value": 95
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 6,
+        "y": 1
+      },
+      "id": 19,
+      "links": [],
+      "options": {
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))",
+          "hide": false,
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "Sys Load (15m avg)",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Non available RAM memory",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgba(50, 172, 45, 0.97)",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 80
+              },
+              {
+                "color": "rgba(245, 54, 54, 0.9)",
+                "value": 90
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 9,
+        "y": 1
+      },
+      "hideTimeOverride": false,
+      "id": 16,
+      "links": [],
+      "options": {
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100",
+          "format": "time_series",
+          "hide": true,
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))",
+          "format": "time_series",
+          "hide": false,
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "B",
+          "step": 240
+        }
+      ],
+      "title": "RAM Used",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Used Swap",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgba(50, 172, 45, 0.97)",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 10
+              },
+              {
+                "color": "rgba(245, 54, 54, 0.9)",
+                "value": 25
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 12,
+        "y": 1
+      },
+      "id": 21,
+      "links": [],
+      "options": {
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100",
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "SWAP Used",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Used Root FS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgba(50, 172, 45, 0.97)",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 80
+              },
+              {
+                "color": "rgba(245, 54, 54, 0.9)",
+                "value": 90
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 15,
+        "y": 1
+      },
+      "id": 154,
+      "links": [],
+      "options": {
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))",
+          "format": "time_series",
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "Root FS Used",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Total number of CPU cores",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 2,
+        "x": 18,
+        "y": 1
+      },
+      "id": 14,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Cores",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "System uptime",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 20,
+        "y": 1
+      },
+      "hideTimeOverride": true,
+      "id": 15,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}",
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "Uptime",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Total RootFS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 0,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "rgba(50, 172, 45, 0.97)",
+                "value": null
+              },
+              {
+                "color": "rgba(237, 129, 40, 0.89)",
+                "value": 70
+              },
+              {
+                "color": "rgba(245, 54, 54, 0.9)",
+                "value": 90
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 2,
+        "x": 18,
+        "y": 3
+      },
+      "id": 23,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}",
+          "format": "time_series",
+          "hide": false,
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "RootFS Total",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Total RAM",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 0,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 2,
+        "x": 20,
+        "y": 3
+      },
+      "id": 75,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}",
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "RAM Total",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Total SWAP",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 0,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 2,
+        "x": 22,
+        "y": 3
+      },
+      "id": 18,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}",
+          "instant": true,
+          "intervalFactor": 1,
+          "range": false,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "SWAP Total",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 5
+      },
+      "id": 263,
+      "panels": [],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Basic CPU / Mem / Net / Disk",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Basic CPU info",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 40,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "percent"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Busy Iowait"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#890F02",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Idle"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#052B51",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Busy Iowait"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#890F02",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Idle"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Busy System"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#EAB839",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Busy User"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A437C",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Busy Other"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#6D1F62",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 77,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "width": 250
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "9.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "legendFormat": "Busy System",
+          "range": true,
+          "refId": "A",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "legendFormat": "Busy User",
+          "range": true,
+          "refId": "B",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Busy Iowait",
+          "range": true,
+          "refId": "C",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Busy IRQs",
+          "range": true,
+          "refId": "D",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Busy Other",
+          "range": true,
+          "refId": "E",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "editorMode": "code",
+          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Idle",
+          "range": true,
+          "refId": "F",
+          "step": 240
+        }
+      ],
+      "title": "CPU Basic",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Basic memory usage",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 40,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Apps"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#629E51",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Buffers"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#614D93",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Cache"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#6D1F62",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Cached"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#511749",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Committed"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#508642",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Free"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A437C",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#CFFAFF",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Inactive"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#584477",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "PageTables"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Page_Tables"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "RAM_Free"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#E0F9D7",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "SWAP Used"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Slab"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#806EB7",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Slab_Cache"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#E0752D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Swap"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Swap Used"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Swap_Cache"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#C15C17",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Swap_Free"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#2F575E",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Unused"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#EAB839",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "RAM Total"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#E0F9D7",
+                  "mode": "fixed"
+                }
+              },
+              {
+                "id": "custom.fillOpacity",
+                "value": 0
+              },
+              {
+                "id": "custom.stacking",
+                "value": {
+                  "group": false,
+                  "mode": "normal"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "RAM Cache + Buffer"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#052B51",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "RAM Free"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Available"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#DEDAF7",
+                  "mode": "fixed"
+                }
+              },
+              {
+                "id": "custom.fillOpacity",
+                "value": 0
+              },
+              {
+                "id": "custom.stacking",
+                "value": {
+                  "group": false,
+                  "mode": "normal"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 78,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "width": 350
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "legendFormat": "RAM Total",
+          "refId": "A",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "legendFormat": "RAM Used",
+          "refId": "B",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "RAM Cache + Buffer",
+          "refId": "C",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "RAM Free",
+          "refId": "D",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "SWAP Used",
+          "refId": "E",
+          "step": 240
+        }
+      ],
+      "title": "Memory Basic",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Basic network info per interface",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 40,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bps"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_bytes_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_drop_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#6ED0E0",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#E0F9D7",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_errs_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#CCA300",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_bytes_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_drop_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#6ED0E0",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#E0F9D7",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_errs_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#CCA300",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_drop_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#99440A",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#967302",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_errs_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#890F02",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_bytes_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_drop_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#99440A",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#967302",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_errs_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#890F02",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/.*trans.*/"
+            },
+            "properties": [
+              {
+                "id": "custom.transform",
+                "value": "negative-Y"
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 13
+      },
+      "id": 74,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "recv {{device}}",
+          "refId": "A",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "trans {{device}} ",
+          "refId": "B",
+          "step": 240
+        }
+      ],
+      "title": "Network Traffic Basic",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P92AEBB27A9B79E22"
+      },
+      "description": "Disk space used of all filesystems mounted",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 40,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 13
+      },
+      "id": 152,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{mountpoint}}",
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "Disk Space Used Basic",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 265,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "percentage",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 70,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "smooth",
+                "lineWidth": 2,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "percent"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Idle - Waiting for something to happen"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Iowait - Waiting for I/O to complete"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Irq - Servicing interrupts"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Nice - Niced processes executing in user mode"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Softirq - Servicing softirqs"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Steal - Time spent in other operating systems when running in a virtualized environment"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCE2DE",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "System - Processes executing in kernel mode"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "User - Normal processes executing in user mode"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#5195CE",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 0,
+            "y": 7
+          },
+          "id": 3,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 250
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "System - Processes executing in kernel mode",
+              "range": true,
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "User - Normal processes executing in user mode",
+              "range": true,
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Nice - Niced processes executing in user mode",
+              "range": true,
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Iowait - Waiting for I/O to complete",
+              "range": true,
+              "refId": "E",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Irq - Servicing interrupts",
+              "range": true,
+              "refId": "F",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Softirq - Servicing softirqs",
+              "range": true,
+              "refId": "G",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment",
+              "range": true,
+              "refId": "H",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Idle - Waiting for something to happen",
+              "range": true,
+              "refId": "J",
+              "step": 240
+            }
+          ],
+          "title": "CPU",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 40,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap - Swap memory usage"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused - Free memory unassigned"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Hardware Corrupted - *./"
+                },
+                "properties": [
+                  {
+                    "id": "custom.stacking",
+                    "value": {
+                      "group": false,
+                      "mode": "normal"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 12,
+            "y": 7
+          },
+          "id": 24,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Apps - Memory used by user-space applications",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)",
+              "refId": "D",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Cache - Parked file data (file content) cache",
+              "refId": "E",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Buffers - Block device (e.g. harddisk) cache",
+              "refId": "F",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Unused - Free memory unassigned",
+              "refId": "G",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Swap - Swap space used",
+              "refId": "H",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working",
+              "refId": "I",
+              "step": 240
+            }
+          ],
+          "title": "Memory Stack",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bits out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 40,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "receive_packets_eth0"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "receive_packets_lo"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "transmit_packets_eth0"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "transmit_packets_lo"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 0,
+            "y": 19
+          },
+          "id": 84,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Transmit",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 40,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 12,
+            "y": 19
+          },
+          "id": 156,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}}",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Disk Space Used",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "IO read (-) / write (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "iops"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Read.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 0,
+            "y": 31
+          },
+          "id": 229,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - Reads completed",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Writes completed",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Disk IOps",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes read (-) / write (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 40,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "Bps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "io time"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*read*./"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byType",
+                  "options": "time"
+                },
+                "properties": [
+                  {
+                    "id": "custom.axisPlacement",
+                    "value": "hidden"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 12,
+            "y": 31
+          },
+          "id": 42,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Successfully read bytes",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Successfully written bytes",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "I/O Usage Read / Write",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "%util",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 40,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "io time"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byType",
+                  "options": "time"
+                },
+                "properties": [
+                  {
+                    "id": "custom.axisPlacement",
+                    "value": "hidden"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 0,
+            "y": 43
+          },
+          "id": 127,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}}",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "I/O Utilization",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "percentage",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 70,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "smooth",
+                "lineWidth": 2,
+                "pointSize": 3,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "max": 1,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/^Guest - /"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#5195ce",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/^GuestNice - /"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#c15c17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 12,
+            "w": 12,
+            "x": 12,
+            "y": 43
+          },
+          "id": 319,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))",
+              "hide": false,
+              "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))",
+              "hide": false,
+              "legendFormat": "GuestNice - Time spent running a niced guest  (virtual CPU for guest operating system)",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "CPU spent seconds in guests (VMs)",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "CPU / Memory / Net / Disk",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 21
+      },
+      "id": 266,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 38
+          },
+          "id": 136,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Inactive - Memory which has been less recently used.  It is more eligible to be reclaimed for other purposes",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Active / Inactive",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*CommitLimit - *./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 38
+          },
+          "id": 135,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Committed_AS - Amount of memory presently allocated on the system",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "CommitLimit - Amount of  memory currently available to be allocated on the system",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Committed",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 48
+          },
+          "id": 191,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Inactive_file - File-backed memory on inactive LRU list",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Active_file - File-backed memory on active LRU list",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs",
+              "refId": "D",
+              "step": 240
+            }
+          ],
+          "title": "Memory Active / Inactive Detail",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 48
+          },
+          "id": 130,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Writeback - Memory which is actively being written back to disk",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Dirty - Memory which is waiting to get written back to the disk",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Memory Writeback and Dirty",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"
+                },
+                "properties": [
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"
+                },
+                "properties": [
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 58
+          },
+          "id": 138,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages",
+              "refId": "D",
+              "step": 240
+            }
+          ],
+          "title": "Memory Shared and Mapped",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 58
+          },
+          "id": 131,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Slab",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 68
+          },
+          "id": 70,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "VmallocTotal - Total size of vmalloc memory area",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "VmallocUsed - Amount of vmalloc area which is used",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Memory Vmalloc",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 68
+          },
+          "id": 159,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Bounce - Memory used for block device bounce buffers",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Memory Bounce",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Inactive *./"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 78
+          },
+          "id": 129,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "AnonHugePages - Memory in anonymous huge pages",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "AnonPages - Memory in user pages not backed by files",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Anonymous",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 78
+          },
+          "id": 160,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Kernel / CPU",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "pages",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 88
+          },
+          "id": 140,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Memory HugePages Counter",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 88
+          },
+          "id": 71,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "HugePages - Total size of the pool of huge pages",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Hugepagesize - Huge Page size",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory HugePages Size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 98
+          },
+          "id": 128,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "DirectMap1G - Amount of pages mapped as this size",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "DirectMap2M - Amount of pages mapped as this size",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "DirectMap4K - Amount of pages mapped as this size",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Memory DirectMap",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 98
+          },
+          "id": 137,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Unevictable and MLocked",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 108
+          },
+          "id": 132,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Memory NFS",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Memory Meminfo",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "id": 267,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "pages out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*out/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "id": 176,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Pagesin - Page in operations",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Pagesout - Page out operations",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Pages In / Out",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "pages out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*out/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "id": 22,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Pswpin - Pages swapped in",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Pswpout - Pages swapped out",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Memory Pages Swap In / Out",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "faults",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Apps"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#629E51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A437C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#CFFAFF",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "RAM_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#806EB7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#2F575E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Unused"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Pgfault - Page major and minor fault operations"
+                },
+                "properties": [
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  },
+                  {
+                    "id": "custom.stacking",
+                    "value": {
+                      "group": false,
+                      "mode": "normal"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 35
+          },
+          "id": 175,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 350
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Pgfault - Page major and minor fault operations",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Pgmajfault - Major page fault operations",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])  - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Pgminfault - Minor page fault operations",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Memory Page Faults",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#99440A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Buffers"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#58140C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6D1F62",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Cached"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Committed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#508642",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Dirty"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Free"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#B7DBAB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Mapped"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PageTables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Page_Tables"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Slab_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Swap_Cache"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C15C17",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#511749",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total RAM + Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#052B51",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Total Swap"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "VmallocUsed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 35
+          },
+          "id": 307,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "oom killer invocations ",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "OOM Killer",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Memory Vmstat",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 23
+      },
+      "id": 293,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Variation*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 40
+          },
+          "id": 260,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Estimated error in seconds",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Time offset in between local system and reference clock",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Maximum error in seconds",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Time Synchronized Drift",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 40
+          },
+          "id": 291,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Phase-locked loop time adjust",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Time PLL Adjust",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Variation*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 50
+          },
+          "id": 168,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Local clock frequency adjustment",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Time Synchronized Status",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 50
+          },
+          "id": 294,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Seconds between clock ticks",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "International Atomic Time (TAI) offset",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Time Misc",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "System Timesync",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 312,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "id": 62,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Processes blocked waiting for I/O to complete",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Processes in runnable state",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Processes Status",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 27
+          },
+          "id": 315,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ state }}",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Processes State",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "forks / sec",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 37
+          },
+          "id": 148,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Processes forks second",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Processes  Forks",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "decbytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Max.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 37
+          },
+          "id": 149,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Processes virtual memory size in bytes",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Maximum amount of virtual memory available in bytes",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Processes virtual memory size in bytes",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Maximum amount of virtual memory available in bytes",
+              "refId": "D",
+              "step": 240
+            }
+          ],
+          "title": "Processes Memory",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "PIDs limit"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F2495C",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 47
+          },
+          "id": 313,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Number of PIDs",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "PIDs limit",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "PIDs Number and Limit",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*waiting.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 47
+          },
+          "id": 305,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "CPU {{ cpu }} - seconds spent running a process",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Process schedule stats Running / Waiting",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Threads limit"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F2495C",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 57
+          },
+          "id": 314,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Allocated threads",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Threads limit",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Threads Number and Limit",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "System Processes",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 25
+      },
+      "id": 269,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 42
+          },
+          "id": 8,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Context switches",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "Interrupts",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Context Switches / Interrupts",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 42
+          },
+          "id": 7,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_load1{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 4,
+              "legendFormat": "Load 1m",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_load5{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 4,
+              "legendFormat": "Load 5m",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_load15{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 4,
+              "legendFormat": "Load 15m",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "System Load",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Critical*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Max*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 52
+          },
+          "id": 259,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ type }} - {{ info }}",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Interrupts Detail",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 52
+          },
+          "id": 306,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "CPU {{ cpu }}",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Schedule timeslices executed by each cpu",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 62
+          },
+          "id": 151,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Entropy available to random number generators",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Entropy",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 62
+          },
+          "id": 308,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Time spent",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "CPU time spent in user and system contexts",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Max*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 72
+          },
+          "id": 64,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Maximum open file descriptors",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Open file descriptors",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "File Descriptors",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "System Misc",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "id": 304,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "temperature",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "celsius"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Critical*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Max*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 43
+          },
+          "id": 158,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ chip }} {{ sensor }} temp",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": true,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ chip }} {{ sensor }} Critical Alarm",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ chip }} {{ sensor }} Critical",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": true,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ chip }} {{ sensor }} Critical Historical",
+              "refId": "D",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": true,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ chip }} {{ sensor }} Max",
+              "refId": "E",
+              "step": 240
+            }
+          ],
+          "title": "Hardware temperature monitor",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Max*./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 43
+          },
+          "id": 300,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Current {{ name }} in {{ type }}",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Max {{ name }} in {{ type }}",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Throttle cooling device",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 53
+          },
+          "id": 302,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ power_supply }} online",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Power supply",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Hardware Misc",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 27
+      },
+      "id": 296,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 30
+          },
+          "id": 297,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ name }} Connections",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Systemd Sockets",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Failed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F2495C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Inactive"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FF9830",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Active"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#73BF69",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Deactivating"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FFCB7D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Activating"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#C8F2C2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 30
+          },
+          "id": 298,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Activating",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Active",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Deactivating",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Failed",
+              "refId": "D",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Inactive",
+              "refId": "E",
+              "step": 240
+            }
+          ],
+          "title": "Systemd Units State",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Systemd",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 270,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "The number (after merges) of I/O requests completed per second for the device",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "IO read (-) / write (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "iops"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Read.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 31
+          },
+          "id": 9,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - Reads completed",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Writes completed",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Disk IOps Completed",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "The number of bytes read from or written to the device per second",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes read (-) / write (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "Bps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Read.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 31
+          },
+          "id": 33,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - Read bytes",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Written bytes",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Disk R/W Data",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "time. read (-) / write (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 30,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Read.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 41
+          },
+          "id": 37,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - Read wait time avg",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Write wait time avg",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Disk Average Wait Time",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "The average queue length of the requests that were issued to the device",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "aqu-sz",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 41
+          },
+          "id": 35,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}}",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Average Queue Size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "The number of read and write requests merged per second that were queued to the device",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "I/Os",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "iops"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Read.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 51
+          },
+          "id": 133,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Read merged",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Write merged",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Disk R/W Merged",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially.  But for devices  serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "%util",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 30,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 51
+          },
+          "id": 36,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - IO",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - discard",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Time Spent Doing I/Os",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "Outstanding req.",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 61
+          },
+          "id": 34,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}",
+              "interval": "",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - IO now",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Instantaneous Queue Size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "IOs",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "iops"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EAB839",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#6ED0E0",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EF843C",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#584477",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda2_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BA43A9",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sda3_.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F4D598",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#0A50A1",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#BF1B00",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdb3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0752D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#962D82",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#614D93",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdc3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#9AC48A",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#65C5DB",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9934E",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#EA6460",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde1.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E0F9D7",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sdd2.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#FCEACA",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*sde3.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F9E2D2",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 61
+          },
+          "id": 301,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "intervalFactor": 4,
+              "legendFormat": "{{device}} - Discards completed",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Discards merged",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Disk IOps Discards completed / merged",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Storage Disk",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 29
+      },
+      "id": 271,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 46
+          },
+          "id": 43,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}} - Available",
+              "metric": "",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
+              "format": "time_series",
+              "hide": true,
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}} - Free",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
+              "format": "time_series",
+              "hide": true,
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}} - Size",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Filesystem space available",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "file nodes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 46
+          },
+          "id": 41,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}} - Free file nodes",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "File Nodes Free",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "files",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 56
+          },
+          "id": 28,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 4,
+              "legendFormat": "Max open files",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "Open files",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "File Descriptor",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "file Nodes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 56
+          },
+          "id": 219,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}} - File nodes total",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "File Nodes Size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "max": 1,
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "/ ReadOnly"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 66
+          },
+          "id": 44,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}} - ReadOnly",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{mountpoint}} - Device error",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Filesystem in ReadOnly / Error",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Storage Filesystem",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 30
+      },
+      "id": 272,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "pps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "receive_packets_eth0"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "receive_packets_lo"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "transmit_packets_eth0"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#7EB26D",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "transmit_packets_lo"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#E24D42",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 31
+          },
+          "id": 60,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Transmit",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic by Packets",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "pps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 31
+          },
+          "id": 142,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive errors",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Rransmit errors",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Errors",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "pps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 41
+          },
+          "id": 143,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive drop",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Transmit drop",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Drop",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "pps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 41
+          },
+          "id": 141,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive compressed",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Transmit compressed",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Compressed",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "pps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 51
+          },
+          "id": 146,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive multicast",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Multicast",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "pps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 51
+          },
+          "id": 144,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive fifo",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Transmit fifo",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Fifo",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "pps"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 61
+          },
+          "id": 145,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Receive frame",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Frame",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 61
+          },
+          "id": 231,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Statistic transmit_carrier",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Carrier",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Trans.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 71
+          },
+          "id": 232,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{device}} - Transmit colls",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Network Traffic Colls",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "entries",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "NF conntrack limit"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 71
+          },
+          "id": 61,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "NF conntrack entries",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "NF conntrack limit",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "NF Contrack",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "Entries",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 81
+          },
+          "id": 230,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{ device }} - ARP entries",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "ARP Entries",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 81
+          },
+          "id": 288,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{ device }} - Bytes",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "MTU",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 91
+          },
+          "id": 280,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{ device }} - Speed",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Speed",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packets",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 91
+          },
+          "id": 289,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{ device }} -   Interface transmit queue length",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Queue Length",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "packetes drop (-) / process (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Dropped.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 101
+          },
+          "id": 290,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "CPU {{cpu}} - Processed",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "CPU {{cpu}} - Dropped",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Softnet Packets",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 101
+          },
+          "id": 310,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "CPU {{cpu}} - Squeezed",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Softnet Out of Quota",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 111
+          },
+          "id": 309,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{interface}} - Operational state UP",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{device}} - Physical link state",
+              "refId": "B"
+            }
+          ],
+          "title": "Network Operational Status",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Network Traffic",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 31
+      },
+      "id": 273,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 32
+          },
+          "id": 63,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "TCP_alloc - Allocated sockets",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "TCP_inuse - Tcp sockets currently in use",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": true,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "TCP_mem - Used memory for tcp",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "TCP_orphan - Orphan sockets",
+              "refId": "D",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "TCP_tw - Sockets waiting close",
+              "refId": "E",
+              "step": 240
+            }
+          ],
+          "title": "Sockstat TCP",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 32
+          },
+          "id": 124,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "UDP_inuse - Udp sockets currently in use",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "UDP_mem - Used memory for udp",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Sockstat UDP",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 42
+          },
+          "id": 125,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "FRAG_inuse - Frag sockets currently in use",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "RAW_inuse - Raw sockets currently in use",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "Sockstat FRAG / RAW",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "bytes",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 42
+          },
+          "id": 220,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "mem_bytes - TCP sockets in that state",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "mem_bytes - UDP sockets in that state",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "FRAG_memory - Used memory for frag",
+              "refId": "C"
+            }
+          ],
+          "title": "Sockstat Memory Size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "sockets",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 52
+          },
+          "id": 126,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Sockets_used - Sockets currently in use",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Sockstat Used",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Network Sockstat",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 32
+      },
+      "id": 274,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "octets out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Out.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 33
+          },
+          "id": 221,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "InOctets - Received octets",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "OutOctets - Sent octets",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Netstat IP In / Out Octets",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "datagrams",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 33
+          },
+          "id": 81,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "width": 300
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Forwarding - IP forwarding",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Netstat IP Forwarding",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "messages out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Out.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 43
+          },
+          "id": 115,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "InMsgs -  Messages which the entity received. Note that this counter includes all those counted by icmpInErrors",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "ICMP In / Out",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "messages out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Out.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 43
+          },
+          "id": 50,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "ICMP Errors",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "datagrams out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Out.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Snd.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 53
+          },
+          "id": 55,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "InDatagrams - Datagrams received",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "OutDatagrams - Datagrams sent",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "UDP In / Out",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "datagrams",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 53
+          },
+          "id": 109,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application",
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "RcvbufErrors - UDP buffer errors received",
+              "refId": "D",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "SndbufErrors - UDP buffer errors send",
+              "refId": "E",
+              "step": 240
+            }
+          ],
+          "title": "UDP Errors",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "datagrams out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Out.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Snd.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 63
+          },
+          "id": 299,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "TCP In / Out",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 63
+          },
+          "id": 104,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits",
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets",
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)",
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "interval": "",
+              "legendFormat": "OutRsts - Segments sent with RST flag",
+              "refId": "F"
+            }
+          ],
+          "title": "TCP Errors",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "connections",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*MaxConn *./"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#890F02",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 73
+          },
+          "id": 85,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "TCP Connections",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter out (-) / in (+)",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*Sent.*/"
+                },
+                "properties": [
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 73
+          },
+          "id": 91,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "SyncookiesFailed - Invalid SYN cookies received",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "SyncookiesRecv - SYN cookies received",
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "SyncookiesSent - SYN cookies sent",
+              "refId": "C",
+              "step": 240
+            }
+          ],
+          "title": "TCP SynCookie",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "connections",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 83
+          },
+          "id": 82,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "TCP Direct Transition",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "Enable with --collector.tcpstat argument on node-exporter",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "connections",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 83
+          },
+          "id": 320,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "established - TCP sockets in established state",
+              "range": true,
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "fin_wait2 - TCP sockets in fin_wait2 state",
+              "range": true,
+              "refId": "B",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "listen - TCP sockets in listen state",
+              "range": true,
+              "refId": "C",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "editorMode": "code",
+              "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "time_wait - TCP sockets in time_wait state",
+              "range": true,
+              "refId": "D",
+              "step": 240
+            }
+          ],
+          "title": "TCP Stat",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Network Netstat",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "000000001"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 33
+      },
+      "id": 279,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 50
+          },
+          "id": 40,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{collector}} - Scrape duration",
+              "refId": "A",
+              "step": 240
+            }
+          ],
+          "title": "Node Exporter Scrape Time",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P92AEBB27A9B79E22"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "counter",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.*error.*/"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "#F2495C",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.transform",
+                    "value": "negative-Y"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 50
+          },
+          "id": 157,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max",
+                "min"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "9.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{collector}} - Scrape success",
+              "refId": "A",
+              "step": 240
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "P92AEBB27A9B79E22"
+              },
+              "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{collector}} - Scrape textfile error (1 = true)",
+              "refId": "B",
+              "step": 240
+            }
+          ],
+          "title": "Node Exporter Scrape",
+          "type": "timeseries"
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "000000001"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Node Exporter",
+      "type": "row"
+    }
+  ],
+  "refresh": "5s",
+  "revision": 1,
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [
+    "linux"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "node",
+          "value": "node"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "P92AEBB27A9B79E22"
+        },
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Job",
+        "multi": false,
+        "name": "job",
+        "options": [],
+        "query": {
+          "query": "label_values(node_uname_info, job)",
+          "refId": "Prometheus-job-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "node-exporter:9100",
+          "value": "node-exporter:9100"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "P92AEBB27A9B79E22"
+        },
+        "definition": "label_values(node_uname_info{job=\"$job\"}, instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Host",
+        "multi": false,
+        "name": "node",
+        "options": [],
+        "query": {
+          "query": "label_values(node_uname_info{job=\"$job\"}, instance)",
+          "refId": "Prometheus-node-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+",
+          "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"
+        },
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "diskdevices",
+        "options": [
+          {
+            "selected": true,
+            "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+",
+            "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"
+          }
+        ],
+        "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+",
+        "skipUrlSync": false,
+        "type": "custom"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Node Exporter Full",
+  "uid": "rYdddlPWk",
+  "version": 2,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/etc/grafana/dashboards/node-exporter-full.min.json b/etc/grafana/dashboards/node-exporter-full.min.json
new file mode 100644
index 00000000..a782ea1d
--- /dev/null
+++ b/etc/grafana/dashboards/node-exporter-full.min.json
@@ -0,0 +1 @@
+{"annotations":{"list":[{"$$hashKey":"object:1058","builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"gnetId":1860,"graphTooltip":1,"id":2,"links":[{"icon":"external link","tags":[],"targetBlank":true,"title":"GitHub","type":"link","url":"https://github.com/rfmoz/grafana-dashboards"},{"icon":"external link","tags":[],"targetBlank":true,"title":"Grafana","type":"link","url":"https://grafana.com/grafana/dashboards/1860"}],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":261,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Quick CPU / Mem / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":0,"y":1},"id":20,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100","hide":false,"instant":true,"intervalFactor":1,"legendFormat":"","range":false,"refId":"A","step":240}],"title":"CPU Busy","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together (5 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":3,"y":1},"id":155,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Sys Load (5m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together (15 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":6,"y":1},"id":19,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Sys Load (15m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Non available RAM memory","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":9,"y":1},"hideTimeOverride":false,"id":16,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100","format":"time_series","hide":true,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"B","step":240}],"title":"RAM Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Used Swap","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":10},{"color":"rgba(245, 54, 54, 0.9)","value":25}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":12,"y":1},"id":21,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"SWAP Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Used Root FS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":15,"y":1},"id":154,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))","format":"time_series","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Root FS Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total number of CPU cores","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":1},"id":14,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))","legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU Cores","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"System uptime","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":2,"w":4,"x":20,"y":1},"hideTimeOverride":true,"id":15,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Uptime","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total RootFS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":70},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":3},"id":23,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"RootFS Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total RAM","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":20,"y":3},"id":75,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"RAM Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total SWAP","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":22,"y":3},"id":18,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"SWAP Total","type":"stat"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":5},"id":263,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Basic CPU / Mem / Net / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic CPU info","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy System"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy User"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Other"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":6},"id":77,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy System","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy User","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Iowait","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy IRQs","range":true,"refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Other","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Idle","range":true,"refId":"F","step":240}],"title":"CPU Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic memory usage","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"SWAP Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Total"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]},{"matcher":{"id":"byName","options":"RAM Cache + Buffer"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Free"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"color","value":{"fixedColor":"#DEDAF7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":7,"w":12,"x":12,"y":6},"id":78,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Total","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Used","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Cache + Buffer","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Free","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","intervalFactor":1,"legendFormat":"SWAP Used","refId":"E","step":240}],"title":"Memory Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic network info per interface","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"Recv_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":13},"id":74,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"recv {{device}}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"trans {{device}} ","refId":"B","step":240}],"title":"Network Traffic Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Disk space used of all filesystems mounted","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":13},"id":152,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used Basic","type":"timeseries"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":20},"id":265,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Idle - Waiting for something to happen"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Iowait - Waiting for I/O to complete"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Irq - Servicing interrupts"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Nice - Niced processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Softirq - Servicing softirqs"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Steal - Time spent in other operating systems when running in a virtualized environment"},"properties":[{"id":"color","value":{"fixedColor":"#FCE2DE","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"System - Processes executing in kernel mode"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"User - Normal processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#5195CE","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":7},"id":3,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"System - Processes executing in kernel mode","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"User - Normal processes executing in user mode","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Nice - Niced processes executing in user mode","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Iowait - Waiting for I/O to complete","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Irq - Servicing interrupts","range":true,"refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Softirq - Servicing softirqs","range":true,"refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Steal - Time spent in other operating systems when running in a virtualized environment","range":true,"refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Idle - Waiting for something to happen","range":true,"refId":"J","step":240}],"title":"CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap - Swap memory usage"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused - Free memory unassigned"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Hardware Corrupted - *./"},"properties":[{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":7},"id":24,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Apps - Memory used by user-space applications","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"PageTables - Memory used to map between virtual and physical memory addresses","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Cache - Parked file data (file content) cache","refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Buffers - Block device (e.g. harddisk) cache","refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Unused - Free memory unassigned","refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Swap - Swap space used","refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working","refId":"I","step":240}],"title":"Memory Stack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bits out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":19},"id":84,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":12,"w":12,"x":12,"y":19},"id":156,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":31},"id":229,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*read*./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":31},"id":42,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully written bytes","refId":"B","step":240}],"title":"I/O Usage Read / Write","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":43},"id":127,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"I/O Utilization","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"bars","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":3,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":1,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/^Guest - /"},"properties":[{"id":"color","value":{"fixedColor":"#5195ce","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/^GuestNice - /"},"properties":[{"id":"color","value":{"fixedColor":"#c15c17","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":43},"id":319,"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"Guest - Time spent running a virtual CPU for a guest operating system","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"GuestNice - Time spent running a niced guest  (virtual CPU for guest operating system)","range":true,"refId":"B"}],"title":"CPU spent seconds in guests (VMs)","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"CPU / Memory / Net / Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":266,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":38},"id":136,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Inactive - Memory which has been less recently used.  It is more eligible to be reclaimed for other purposes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary","refId":"B","step":240}],"title":"Memory Active / Inactive","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*CommitLimit - *./"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":38},"id":135,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Committed_AS - Amount of memory presently allocated on the system","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"CommitLimit - Amount of  memory currently available to be allocated on the system","refId":"B","step":240}],"title":"Memory Committed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":48},"id":191,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_file - File-backed memory on inactive LRU list","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_file - File-backed memory on active LRU list","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs","refId":"D","step":240}],"title":"Memory Active / Inactive Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":48},"id":130,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Writeback - Memory which is actively being written back to disk","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"WritebackTmp - Memory used by FUSE for temporary writeback buffers","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Dirty - Memory which is waiting to get written back to the disk","refId":"C","step":240}],"title":"Memory Writeback and Dirty","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":58},"id":138,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Mapped - Used memory in mapped pages files which have been mapped, such as libraries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Shmem - Used shared memory (shared between several processes, thus including RAM disks)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages","refId":"D","step":240}],"title":"Memory Shared and Mapped","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":58},"id":131,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SReclaimable - Part of Slab, that might be reclaimed, such as caches","refId":"B","step":240}],"title":"Memory Slab","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":68},"id":70,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocChunk - Largest contiguous block of vmalloc area which is free","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocTotal - Total size of vmalloc memory area","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocUsed - Amount of vmalloc area which is used","refId":"C","step":240}],"title":"Memory Vmalloc","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":68},"id":159,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Bounce - Memory used for block device bounce buffers","refId":"A","step":240}],"title":"Memory Bounce","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Inactive *./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":78},"id":129,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonHugePages - Memory in anonymous huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonPages - Memory in user pages not backed by files","refId":"B","step":240}],"title":"Memory Anonymous","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":78},"id":160,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"KernelStack - Kernel memory stack. This is not reclaimable","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PerCPU - Per CPU memory allocated dynamically by loadable modules","refId":"B","step":240}],"title":"Memory Kernel / CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":88},"id":140,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Free - Huge pages in the pool that are not yet allocated","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages","refId":"C","step":240}],"title":"Memory HugePages Counter","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":88},"id":71,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages - Total size of the pool of huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Hugepagesize - Huge Page size","refId":"B","step":240}],"title":"Memory HugePages Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":98},"id":128,"links":[],"options":{"legend":{"calcs":["mean","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"DirectMap1G - Amount of pages mapped as this size","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap2M - Amount of pages mapped as this size","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap4K - Amount of pages mapped as this size","refId":"C","step":240}],"title":"Memory DirectMap","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":98},"id":137,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"MLocked - Size of pages locked to memory using the mlock() system call","refId":"B","step":240}],"title":"Memory Unevictable and MLocked","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":108},"id":132,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage","refId":"A","step":240}],"title":"Memory NFS","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Memory Meminfo","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":267,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":25},"id":176,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesin - Page in operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesout - Page out operations","refId":"B","step":240}],"title":"Memory Pages In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":25},"id":22,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpin - Pages swapped in","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpout - Pages swapped out","refId":"B","step":240}],"title":"Memory Pages Swap In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"faults","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Pgfault - Page major and minor fault operations"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":35},"id":175,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgfault - Page major and minor fault operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgmajfault - Major page fault operations","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])  - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgminfault - Minor page fault operations","refId":"C","step":240}],"title":"Memory Page Faults","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":35},"id":307,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"oom killer invocations ","refId":"A","step":240}],"title":"OOM Killer","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Memory Vmstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":23},"id":293,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":40},"id":260,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Estimated error in seconds","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Time offset in between local system and reference clock","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum error in seconds","refId":"C","step":240}],"title":"Time Synchronized Drift","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":40},"id":291,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Phase-locked loop time adjust","refId":"A","step":240}],"title":"Time PLL Adjust","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":168,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_sync_status{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Is clock synchronized to a reliable server (1 = yes, 0 = no)","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Local clock frequency adjustment","refId":"B","step":240}],"title":"Time Synchronized Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":294,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Seconds between clock ticks","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"International Atomic Time (TAI) offset","refId":"B","step":240}],"title":"Time Misc","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Timesync","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":24},"id":312,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":27},"id":62,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_procs_blocked{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes blocked waiting for I/O to complete","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_procs_running{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes in runnable state","refId":"B","step":240}],"title":"Processes Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":27},"id":315,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ state }}","refId":"A","step":240}],"title":"Processes State","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"forks / sec","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":37},"id":148,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Processes forks second","refId":"A","step":240}],"title":"Processes  Forks","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max.*/"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":37},"id":149,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"D","step":240}],"title":"Processes Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"PIDs limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":47},"id":313,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_pids{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Number of PIDs","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_max_processes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PIDs limit","refId":"B","step":240}],"title":"PIDs Number and Limit","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*waiting.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":47},"id":305,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent running a process","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent by processing waiting for this CPU","refId":"B","step":240}],"title":"Process schedule stats Running / Waiting","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Threads limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":57},"id":314,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Allocated threads","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_max_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Threads limit","refId":"B","step":240}],"title":"Threads Number and Limit","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Processes","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":25},"id":269,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":8,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Context switches","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Interrupts","refId":"B","step":240}],"title":"Context Switches / Interrupts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":7,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load1{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 1m","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load5{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 5m","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load15{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 15m","refId":"C","step":240}],"title":"System Load","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":259,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ type }} - {{ info }}","refId":"A","step":240}],"title":"Interrupts Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":52},"id":306,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }}","refId":"A","step":240}],"title":"Schedule timeslices executed by each cpu","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":62},"id":151,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_entropy_available_bits{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Entropy available to random number generators","refId":"A","step":240}],"title":"Entropy","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":62},"id":308,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Time spent","refId":"A","step":240}],"title":"CPU time spent in user and system contexts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":72},"id":64,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_max_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Maximum open file descriptors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_open_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Open file descriptors","refId":"B","step":240}],"title":"File Descriptors","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":26},"id":304,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"temperature","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"celsius"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":158,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} temp","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Alarm","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Historical","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Max","refId":"E","step":240}],"title":"Hardware temperature monitor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":300,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Current {{ name }} in {{ type }}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Max {{ name }} in {{ type }}","refId":"B","step":240}],"title":"Throttle cooling device","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":302,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_power_supply_online{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{ power_supply }} online","refId":"A","step":240}],"title":"Power supply","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Hardware Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":296,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":30},"id":297,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ name }} Connections","refId":"A","step":240}],"title":"Systemd Sockets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Failed"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#FF9830","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#73BF69","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Deactivating"},"properties":[{"id":"color","value":{"fixedColor":"#FFCB7D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Activating"},"properties":[{"id":"color","value":{"fixedColor":"#C8F2C2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":30},"id":298,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Activating","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Active","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Deactivating","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Failed","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Inactive","refId":"E","step":240}],"title":"Systemd Units State","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Systemd","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":28},"id":270,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number (after merges) of I/O requests completed per second for the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":9,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps Completed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of bytes read from or written to the device per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":33,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":4,"legendFormat":"{{device}} - Read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Written bytes","refId":"B","step":240}],"title":"Disk R/W Data","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"time. read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":41},"id":37,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":4,"legendFormat":"{{device}} - Read wait time avg","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}} - Write wait time avg","refId":"B","step":240}],"title":"Disk Average Wait Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The average queue length of the requests that were issued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"aqu-sz","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":41},"id":35,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"Average Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of read and write requests merged per second that were queued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"I/Os","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":51},"id":133,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Read merged","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Write merged","refId":"B","step":240}],"title":"Disk R/W Merged","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially.  But for devices  serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":51},"id":36,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - discard","refId":"B","step":240}],"title":"Time Spent Doing I/Os","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Outstanding req.","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":61},"id":34,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_disk_io_now{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO now","refId":"A","step":240}],"title":"Instantaneous Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IOs","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":61},"id":301,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - Discards completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Discards merged","refId":"B","step":240}],"title":"Disk IOps Discards completed / merged","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Storage Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":271,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":46},"id":43,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Available","metric":"","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Size","refId":"C","step":240}],"title":"Filesystem space available","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":46},"id":41,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free file nodes","refId":"A","step":240}],"title":"File Nodes Free","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"files","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":56},"id":28,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filefd_maximum{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Max open files","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filefd_allocated{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Open files","refId":"B","step":240}],"title":"File Descriptor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file Nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":56},"id":219,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - File nodes total","refId":"A","step":240}],"title":"File Nodes Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"/ ReadOnly"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":66},"id":44,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}} - ReadOnly","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{mountpoint}} - Device error","refId":"B","step":240}],"title":"Filesystem in ReadOnly / Error","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Storage Filesystem","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":272,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":60,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic by Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":142,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive errors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Rransmit errors","refId":"B","step":240}],"title":"Network Traffic Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":41},"id":143,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive drop","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit drop","refId":"B","step":240}],"title":"Network Traffic Drop","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":41},"id":141,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive compressed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit compressed","refId":"B","step":240}],"title":"Network Traffic Compressed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":51},"id":146,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive multicast","refId":"A","step":240}],"title":"Network Traffic Multicast","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":51},"id":144,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive fifo","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit fifo","refId":"B","step":240}],"title":"Network Traffic Fifo","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":61},"id":145,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Receive frame","refId":"A","step":240}],"title":"Network Traffic Frame","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":61},"id":231,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Statistic transmit_carrier","refId":"A","step":240}],"title":"Network Traffic Carrier","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":71},"id":232,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit colls","refId":"A","step":240}],"title":"Network Traffic Colls","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"NF conntrack limit"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":71},"id":61,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack entries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack limit","refId":"B","step":240}],"title":"NF Contrack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":81},"id":230,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_arp_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - ARP entries","refId":"A","step":240}],"title":"ARP Entries","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":81},"id":288,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Bytes","refId":"A","step":240}],"title":"MTU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":91},"id":280,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_speed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Speed","refId":"A","step":240}],"title":"Speed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":91},"id":289,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} -   Interface transmit queue length","refId":"A","step":240}],"title":"Queue Length","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packetes drop (-) / process (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Dropped.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":101},"id":290,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Processed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Dropped","refId":"B","step":240}],"title":"Softnet Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":101},"id":310,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Squeezed","refId":"A","step":240}],"title":"Softnet Out of Quota","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":111},"id":309,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{interface}} - Operational state UP","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_carrier{instance=\"$node\",job=\"$job\"}","format":"time_series","instant":false,"legendFormat":"{{device}} - Physical link state","refId":"B"}],"title":"Network Operational Status","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Traffic","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":273,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":32},"id":63,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_alloc - Allocated sockets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_inuse - Tcp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"TCP_mem - Used memory for tcp","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_orphan - Orphan sockets","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_tw - Sockets waiting close","refId":"E","step":240}],"title":"Sockstat TCP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":32},"id":124,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDPLITE_inuse - Udplite sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_inuse - Udp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_mem - Used memory for udp","refId":"C","step":240}],"title":"Sockstat UDP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":125,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"FRAG_inuse - Frag sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RAW_inuse - Raw sockets currently in use","refId":"C","step":240}],"title":"Sockstat FRAG / RAW","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":220,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - TCP sockets in that state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - UDP sockets in that state","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"FRAG_memory - Used memory for frag","refId":"C"}],"title":"Sockstat Memory Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"sockets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":126,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Sockets_used - Sockets currently in use","refId":"A","step":240}],"title":"Sockstat Used","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Sockstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":32},"id":274,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"octets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":33},"id":221,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InOctets - Received octets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"OutOctets - Sent octets","refId":"B","step":240}],"title":"Netstat IP In / Out Octets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":33},"id":81,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Forwarding - IP forwarding","refId":"A","step":240}],"title":"Netstat IP Forwarding","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":115,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InMsgs -  Messages which the entity received. Note that this counter includes all those counted by icmpInErrors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors","refId":"B","step":240}],"title":"ICMP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":50,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)","refId":"A","step":240}],"title":"ICMP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":55,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InDatagrams - Datagrams received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutDatagrams - Datagrams sent","refId":"B","step":240}],"title":"UDP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":53},"id":109,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - UDP Datagrams that could not be delivered to an application","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"NoPorts - UDP Datagrams received on a port with no listener","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrors Lite - UDPLite Datagrams that could not be delivered to an application","refId":"C"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RcvbufErrors - UDP buffer errors received","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"SndbufErrors - UDP buffer errors send","refId":"E","step":240}],"title":"UDP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":63},"id":299,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","instant":false,"interval":"","intervalFactor":1,"legendFormat":"InSegs - Segments received, including those received in error. This count includes segments received on currently established connections","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets","refId":"B","step":240}],"title":"TCP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":63},"id":104,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenOverflows - Times the listen queue of a socket overflowed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenDrops - SYNs to LISTEN sockets ignored","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets","refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrs - Segments received in error (e.g., bad TCP checksums)","refId":"E"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"OutRsts - Segments sent with RST flag","refId":"F"}],"title":"TCP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*MaxConn *./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":73},"id":85,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")","refId":"B","step":240}],"title":"TCP Connections","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Sent.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":73},"id":91,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesFailed - Invalid SYN cookies received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesRecv - SYN cookies received","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesSent - SYN cookies sent","refId":"C","step":240}],"title":"TCP SynCookie","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":83},"id":82,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state","refId":"B","step":240}],"title":"TCP Direct Transition","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Enable with --collector.tcpstat argument on node-exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":83},"id":320,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"established - TCP sockets in established state","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"fin_wait2 - TCP sockets in fin_wait2 state","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"listen - TCP sockets in listen state","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"time_wait - TCP sockets in time_wait state","range":true,"refId":"D","step":240}],"title":"TCP Stat","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Netstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":33},"id":279,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":40,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape duration","refId":"A","step":240}],"title":"Node Exporter Scrape Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineStyle":{"fill":"solid"},"lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*error.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":157,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_scrape_collector_success{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape success","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape textfile error (1 = true)","refId":"B","step":240}],"title":"Node Exporter Scrape","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Node Exporter","type":"row"}],"refresh":"5s","revision":1,"schemaVersion":38,"style":"dark","tags":["linux"],"templating":{"list":[{"current":{"selected":false,"text":"default","value":"default"},"hide":0,"includeAll":false,"label":"datasource","multi":false,"name":"DS_PROMETHEUS","options":[],"query":"prometheus","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":false,"text":"node","value":"node"},"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"definition":"","hide":0,"includeAll":false,"label":"Job","multi":false,"name":"job","options":[],"query":{"query":"label_values(node_uname_info, job)","refId":"Prometheus-job-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"node-exporter:9100","value":"node-exporter:9100"},"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"definition":"label_values(node_uname_info{job=\"$job\"}, instance)","hide":0,"includeAll":false,"label":"Host","multi":false,"name":"node","options":[],"query":{"query":"label_values(node_uname_info{job=\"$job\"}, instance)","refId":"Prometheus-node-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"},"hide":2,"includeAll":false,"multi":false,"name":"diskdevices","options":[{"selected":true,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"}],"query":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"Node Exporter Full","uid":"rYdddlPWk","version":2,"weekStart":""}
diff --git a/etc/grafana/provisioning/dashboards/foyer.yml b/etc/grafana/provisioning/dashboards/foyer.yml
index 8c23037a..38812c65 100644
--- a/etc/grafana/provisioning/dashboards/foyer.yml
+++ b/etc/grafana/provisioning/dashboards/foyer.yml
@@ -6,4 +6,10 @@ providers:
     allowUiUpdates: false
     options:
       path: /var/lib/grafana/dashboards/foyer.json
-      foldersFromFilesStructure: true
\ No newline at end of file
+      foldersFromFilesStructure: false
+  - name: "node exporter full provider"
+    disableDeletion: true
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards/node-exporter-full.json
+      foldersFromFilesStructure: false
\ No newline at end of file
diff --git a/scripts/minimize-dashboards.sh b/scripts/minimize-dashboards.sh
new file mode 100755
index 00000000..e44face7
--- /dev/null
+++ b/scripts/minimize-dashboards.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# You will need to install jq to use this tool.
+# brew install jq
+
+set -e
+
+DIR="etc/grafana/dashboards"
+
+
+for dashboard in "${DIR}"/*.json; do
+    if [[ "$dashboard" == *.min.json ]]; then
+        continue
+    fi
+    name=$(basename "$dashboard" .json)
+    jq -c < "$dashboard" > "${DIR}/${name}.min.json"
+done
+
+if [ "$1" == "--check" ] ; then
+ if ! git diff --exit-code; then
+    echo "Please run minimize-dashboards.sh and commit after editing the grafana dashboards."
+    exit 1
+ fi
+fi
\ No newline at end of file

From 0a43b7225b47b3fdd6b41a8bfde15cdfe3d95c39 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 26 Oct 2023 18:01:15 +0800
Subject: [PATCH 153/261] chore: only keep minized dashboard (#188)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 etc/grafana/dashboards/foyer.json             |   821 +-
 etc/grafana/dashboards/foyer.min.json         |     1 -
 .../dashboards/node-exporter-full.json        | 23349 +---------------
 .../dashboards/node-exporter-full.min.json    |     1 -
 scripts/minimize-dashboards.sh                |     6 +-
 5 files changed, 7 insertions(+), 24171 deletions(-)
 delete mode 100644 etc/grafana/dashboards/foyer.min.json
 delete mode 100644 etc/grafana/dashboards/node-exporter-full.min.json

diff --git a/etc/grafana/dashboards/foyer.json b/etc/grafana/dashboards/foyer.json
index eed4edd5..50d5f8eb 100644
--- a/etc/grafana/dashboards/foyer.json
+++ b/etc/grafana/dashboards/foyer.json
@@ -1,820 +1 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": 2,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 8,
-      "panels": [],
-      "title": "Storage",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "ops"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "id": 1,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)",
-          "instant": false,
-          "legendFormat": "{{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Op",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "ops"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 1
-      },
-      "id": 2,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)",
-          "instant": false,
-          "legendFormat": "{{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Slow Op",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "s"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 9
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "instant": false,
-          "legendFormat": "p50 - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "p90 - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "p99 - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "pmax - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "D"
-        }
-      ],
-      "title": "Op Duration",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "s"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 9
-      },
-      "id": 4,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "instant": false,
-          "legendFormat": "p50 - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "p90 - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "p99 - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "pmax - {{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "D"
-        }
-      ],
-      "title": "Slow Op Duration",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "Bps"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 17
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ",
-          "instant": false,
-          "legendFormat": "{{foyer}} foyer storage - {{op}} {{extra}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Op Thoughput",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "decbytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 17
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "sum(foyer_storage_total_bytes) by (foyer) ",
-          "instant": false,
-          "legendFormat": "{{foyer}} foyer storage",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Size",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percentunit"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 25
-      },
-      "id": 7,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "a2641a73-8591-446b-9d69-7869ebf43899"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ",
-          "instant": false,
-          "legendFormat": "{{foyer}} foyer storage",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Hit Ratio",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": [],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-30m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "foyer",
-  "uid": "f0e2058b-b292-457c-8ddf-9dbdf7c60035",
-  "version": 2,
-  "weekStart": ""
-}
\ No newline at end of file
+{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":2,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":2,"weekStart":""}
diff --git a/etc/grafana/dashboards/foyer.min.json b/etc/grafana/dashboards/foyer.min.json
deleted file mode 100644
index 50d5f8eb..00000000
--- a/etc/grafana/dashboards/foyer.min.json
+++ /dev/null
@@ -1 +0,0 @@
-{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":2,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":2,"weekStart":""}
diff --git a/etc/grafana/dashboards/node-exporter-full.json b/etc/grafana/dashboards/node-exporter-full.json
index 05527cb9..a782ea1d 100644
--- a/etc/grafana/dashboards/node-exporter-full.json
+++ b/etc/grafana/dashboards/node-exporter-full.json
@@ -1,23348 +1 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "$$hashKey": "object:1058",
-        "builtIn": 1,
-        "datasource": {
-          "type": "datasource",
-          "uid": "grafana"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": [],
-          "type": "dashboard"
-        },
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "gnetId": 1860,
-  "graphTooltip": 1,
-  "id": 2,
-  "links": [
-    {
-      "icon": "external link",
-      "tags": [],
-      "targetBlank": true,
-      "title": "GitHub",
-      "type": "link",
-      "url": "https://github.com/rfmoz/grafana-dashboards"
-    },
-    {
-      "icon": "external link",
-      "tags": [],
-      "targetBlank": true,
-      "title": "Grafana",
-      "type": "link",
-      "url": "https://grafana.com/grafana/dashboards/1860"
-    }
-  ],
-  "liveNow": false,
-  "panels": [
-    {
-      "collapsed": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 261,
-      "panels": [],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Quick CPU / Mem / Disk",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Busy state of all CPU cores together",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 1,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(50, 172, 45, 0.97)",
-                "value": null
-              },
-              {
-                "color": "rgba(237, 129, 40, 0.89)",
-                "value": 85
-              },
-              {
-                "color": "rgba(245, 54, 54, 0.9)",
-                "value": 95
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 3,
-        "x": 0,
-        "y": 1
-      },
-      "id": 20,
-      "links": [],
-      "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100",
-          "hide": false,
-          "instant": true,
-          "intervalFactor": 1,
-          "legendFormat": "",
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "CPU Busy",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Busy state of all CPU cores together (5 min average)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 1,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(50, 172, 45, 0.97)",
-                "value": null
-              },
-              {
-                "color": "rgba(237, 129, 40, 0.89)",
-                "value": 85
-              },
-              {
-                "color": "rgba(245, 54, 54, 0.9)",
-                "value": 95
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 3,
-        "x": 3,
-        "y": 1
-      },
-      "id": 155,
-      "links": [],
-      "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))",
-          "format": "time_series",
-          "hide": false,
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "Sys Load (5m avg)",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Busy state of all CPU cores together (15 min average)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 1,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(50, 172, 45, 0.97)",
-                "value": null
-              },
-              {
-                "color": "rgba(237, 129, 40, 0.89)",
-                "value": 85
-              },
-              {
-                "color": "rgba(245, 54, 54, 0.9)",
-                "value": 95
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 3,
-        "x": 6,
-        "y": 1
-      },
-      "id": 19,
-      "links": [],
-      "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))",
-          "hide": false,
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "Sys Load (15m avg)",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Non available RAM memory",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 1,
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(50, 172, 45, 0.97)",
-                "value": null
-              },
-              {
-                "color": "rgba(237, 129, 40, 0.89)",
-                "value": 80
-              },
-              {
-                "color": "rgba(245, 54, 54, 0.9)",
-                "value": 90
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 3,
-        "x": 9,
-        "y": 1
-      },
-      "hideTimeOverride": false,
-      "id": 16,
-      "links": [],
-      "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100",
-          "format": "time_series",
-          "hide": true,
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))",
-          "format": "time_series",
-          "hide": false,
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "B",
-          "step": 240
-        }
-      ],
-      "title": "RAM Used",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Used Swap",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 1,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(50, 172, 45, 0.97)",
-                "value": null
-              },
-              {
-                "color": "rgba(237, 129, 40, 0.89)",
-                "value": 10
-              },
-              {
-                "color": "rgba(245, 54, 54, 0.9)",
-                "value": 25
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 3,
-        "x": 12,
-        "y": 1
-      },
-      "id": 21,
-      "links": [],
-      "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100",
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "SWAP Used",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Used Root FS",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 1,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(50, 172, 45, 0.97)",
-                "value": null
-              },
-              {
-                "color": "rgba(237, 129, 40, 0.89)",
-                "value": 80
-              },
-              {
-                "color": "rgba(245, 54, 54, 0.9)",
-                "value": 90
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 3,
-        "x": 15,
-        "y": 1
-      },
-      "id": 154,
-      "links": [],
-      "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))",
-          "format": "time_series",
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "Root FS Used",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Total number of CPU cores",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 2,
-        "w": 2,
-        "x": 18,
-        "y": 1
-      },
-      "id": 14,
-      "links": [],
-      "maxDataPoints": 100,
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))",
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "CPU Cores",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "System uptime",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 1,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "s"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 2,
-        "w": 4,
-        "x": 20,
-        "y": 1
-      },
-      "hideTimeOverride": true,
-      "id": 15,
-      "links": [],
-      "maxDataPoints": 100,
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}",
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "Uptime",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Total RootFS",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 0,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(50, 172, 45, 0.97)",
-                "value": null
-              },
-              {
-                "color": "rgba(237, 129, 40, 0.89)",
-                "value": 70
-              },
-              {
-                "color": "rgba(245, 54, 54, 0.9)",
-                "value": 90
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 2,
-        "w": 2,
-        "x": 18,
-        "y": 3
-      },
-      "id": 23,
-      "links": [],
-      "maxDataPoints": 100,
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}",
-          "format": "time_series",
-          "hide": false,
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "RootFS Total",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Total RAM",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 0,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 2,
-        "w": 2,
-        "x": 20,
-        "y": 3
-      },
-      "id": 75,
-      "links": [],
-      "maxDataPoints": 100,
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}",
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "RAM Total",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Total SWAP",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 0,
-          "mappings": [
-            {
-              "options": {
-                "match": "null",
-                "result": {
-                  "text": "N/A"
-                }
-              },
-              "type": "special"
-            }
-          ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 2,
-        "w": 2,
-        "x": 22,
-        "y": 3
-      },
-      "id": 18,
-      "links": [],
-      "maxDataPoints": 100,
-      "options": {
-        "colorMode": "none",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.1.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "exemplar": false,
-          "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}",
-          "instant": true,
-          "intervalFactor": 1,
-          "range": false,
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "SWAP Total",
-      "type": "stat"
-    },
-    {
-      "collapsed": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 5
-      },
-      "id": 263,
-      "panels": [],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Basic CPU / Mem / Net / Disk",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Basic CPU info",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 40,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "percent"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "links": [],
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percentunit"
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Busy Iowait"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#890F02",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Idle"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#052B51",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Busy Iowait"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#890F02",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Idle"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#7EB26D",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Busy System"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#EAB839",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Busy User"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A437C",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Busy Other"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#6D1F62",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 0,
-        "y": 6
-      },
-      "id": 77,
-      "links": [],
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true,
-          "width": 250
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "pluginVersion": "9.2.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-          "format": "time_series",
-          "hide": false,
-          "intervalFactor": 1,
-          "legendFormat": "Busy System",
-          "range": true,
-          "refId": "A",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-          "format": "time_series",
-          "hide": false,
-          "intervalFactor": 1,
-          "legendFormat": "Busy User",
-          "range": true,
-          "refId": "B",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "Busy Iowait",
-          "range": true,
-          "refId": "C",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "Busy IRQs",
-          "range": true,
-          "refId": "D",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "Busy Other",
-          "range": true,
-          "refId": "E",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "editorMode": "code",
-          "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "Idle",
-          "range": true,
-          "refId": "F",
-          "step": 240
-        }
-      ],
-      "title": "CPU Basic",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Basic memory usage",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 40,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "normal"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "links": [],
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Apps"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#629E51",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Buffers"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#614D93",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Cache"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#6D1F62",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Cached"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#511749",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Committed"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#508642",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Free"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A437C",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#CFFAFF",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Inactive"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#584477",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "PageTables"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A50A1",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Page_Tables"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A50A1",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "RAM_Free"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#E0F9D7",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "SWAP Used"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#BF1B00",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Slab"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#806EB7",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Slab_Cache"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#E0752D",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Swap"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#BF1B00",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Swap Used"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#BF1B00",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Swap_Cache"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#C15C17",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Swap_Free"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#2F575E",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Unused"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#EAB839",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "RAM Total"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#E0F9D7",
-                  "mode": "fixed"
-                }
-              },
-              {
-                "id": "custom.fillOpacity",
-                "value": 0
-              },
-              {
-                "id": "custom.stacking",
-                "value": {
-                  "group": false,
-                  "mode": "normal"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "RAM Cache + Buffer"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#052B51",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "RAM Free"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#7EB26D",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Available"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#DEDAF7",
-                  "mode": "fixed"
-                }
-              },
-              {
-                "id": "custom.fillOpacity",
-                "value": 0
-              },
-              {
-                "id": "custom.stacking",
-                "value": {
-                  "group": false,
-                  "mode": "normal"
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 78,
-      "links": [],
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true,
-          "width": 350
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "9.2.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}",
-          "format": "time_series",
-          "hide": false,
-          "intervalFactor": 1,
-          "legendFormat": "RAM Total",
-          "refId": "A",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})",
-          "format": "time_series",
-          "hide": false,
-          "intervalFactor": 1,
-          "legendFormat": "RAM Used",
-          "refId": "B",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "RAM Cache + Buffer",
-          "refId": "C",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "RAM Free",
-          "refId": "D",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "SWAP Used",
-          "refId": "E",
-          "step": 240
-        }
-      ],
-      "title": "Memory Basic",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Basic network info per interface",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 40,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "links": [],
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bps"
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Recv_bytes_eth2"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#7EB26D",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Recv_bytes_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A50A1",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Recv_drop_eth2"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#6ED0E0",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Recv_drop_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#E0F9D7",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Recv_errs_eth2"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#BF1B00",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Recv_errs_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#CCA300",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Trans_bytes_eth2"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#7EB26D",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Trans_bytes_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A50A1",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Trans_drop_eth2"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#6ED0E0",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Trans_drop_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#E0F9D7",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Trans_errs_eth2"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#BF1B00",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Trans_errs_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#CCA300",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "recv_bytes_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A50A1",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "recv_drop_eth0"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#99440A",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "recv_drop_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#967302",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "recv_errs_eth0"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#BF1B00",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "recv_errs_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#890F02",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "trans_bytes_eth0"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#7EB26D",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "trans_bytes_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#0A50A1",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "trans_drop_eth0"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#99440A",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "trans_drop_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#967302",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "trans_errs_eth0"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#BF1B00",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "trans_errs_lo"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "#890F02",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byRegexp",
-              "options": "/.*trans.*/"
-            },
-            "properties": [
-              {
-                "id": "custom.transform",
-                "value": "negative-Y"
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 0,
-        "y": 13
-      },
-      "id": 74,
-      "links": [],
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "9.2.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "recv {{device}}",
-          "refId": "A",
-          "step": 240
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "trans {{device}} ",
-          "refId": "B",
-          "step": 240
-        }
-      ],
-      "title": "Network Traffic Basic",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P92AEBB27A9B79E22"
-      },
-      "description": "Disk space used of all filesystems mounted",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 40,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "links": [],
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 12,
-        "y": 13
-      },
-      "id": 152,
-      "links": [],
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "9.2.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "{{mountpoint}}",
-          "refId": "A",
-          "step": 240
-        }
-      ],
-      "title": "Disk Space Used Basic",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 20
-      },
-      "id": 265,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "percentage",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 70,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "smooth",
-                "lineWidth": 2,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "percent"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "percentunit"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Idle - Waiting for something to happen"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Iowait - Waiting for I/O to complete"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Irq - Servicing interrupts"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Nice - Niced processes executing in user mode"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Softirq - Servicing softirqs"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Steal - Time spent in other operating systems when running in a virtualized environment"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCE2DE",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "System - Processes executing in kernel mode"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "User - Normal processes executing in user mode"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#5195CE",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 0,
-            "y": 7
-          },
-          "id": 3,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 250
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "System - Processes executing in kernel mode",
-              "range": true,
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "User - Normal processes executing in user mode",
-              "range": true,
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Nice - Niced processes executing in user mode",
-              "range": true,
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Iowait - Waiting for I/O to complete",
-              "range": true,
-              "refId": "E",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Irq - Servicing interrupts",
-              "range": true,
-              "refId": "F",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Softirq - Servicing softirqs",
-              "range": true,
-              "refId": "G",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment",
-              "range": true,
-              "refId": "H",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Idle - Waiting for something to happen",
-              "range": true,
-              "refId": "J",
-              "step": 240
-            }
-          ],
-          "title": "CPU",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 40,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap - Swap memory usage"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused - Free memory unassigned"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Hardware Corrupted - *./"
-                },
-                "properties": [
-                  {
-                    "id": "custom.stacking",
-                    "value": {
-                      "group": false,
-                      "mode": "normal"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 12,
-            "y": 7
-          },
-          "id": 24,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Apps - Memory used by user-space applications",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)",
-              "refId": "D",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Cache - Parked file data (file content) cache",
-              "refId": "E",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Buffers - Block device (e.g. harddisk) cache",
-              "refId": "F",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Unused - Free memory unassigned",
-              "refId": "G",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Swap - Swap space used",
-              "refId": "H",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working",
-              "refId": "I",
-              "step": 240
-            }
-          ],
-          "title": "Memory Stack",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bits out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 40,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "receive_packets_eth0"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "receive_packets_lo"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "transmit_packets_eth0"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "transmit_packets_lo"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 0,
-            "y": 19
-          },
-          "id": 84,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Transmit",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 40,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 12,
-            "y": 19
-          },
-          "id": 156,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}}",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Disk Space Used",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "IO read (-) / write (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "iops"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Read.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 0,
-            "y": 31
-          },
-          "id": 229,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - Reads completed",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Writes completed",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Disk IOps",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes read (-) / write (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 40,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "Bps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "io time"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*read*./"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byType",
-                  "options": "time"
-                },
-                "properties": [
-                  {
-                    "id": "custom.axisPlacement",
-                    "value": "hidden"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 12,
-            "y": 31
-          },
-          "id": 42,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Successfully read bytes",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Successfully written bytes",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "I/O Usage Read / Write",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "%util",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 40,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "percentunit"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "io time"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byType",
-                  "options": "time"
-                },
-                "properties": [
-                  {
-                    "id": "custom.axisPlacement",
-                    "value": "hidden"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 0,
-            "y": 43
-          },
-          "id": 127,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}}",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "I/O Utilization",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "percentage",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "bars",
-                "fillOpacity": 70,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "smooth",
-                "lineWidth": 2,
-                "pointSize": 3,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "mappings": [],
-              "max": 1,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "percentunit"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/^Guest - /"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#5195ce",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/^GuestNice - /"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#c15c17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 12,
-            "y": 43
-          },
-          "id": 319,
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))",
-              "hide": false,
-              "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system",
-              "range": true,
-              "refId": "A"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))",
-              "hide": false,
-              "legendFormat": "GuestNice - Time spent running a niced guest  (virtual CPU for guest operating system)",
-              "range": true,
-              "refId": "B"
-            }
-          ],
-          "title": "CPU spent seconds in guests (VMs)",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "CPU / Memory / Net / Disk",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 21
-      },
-      "id": 266,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 38
-          },
-          "id": 136,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Inactive - Memory which has been less recently used.  It is more eligible to be reclaimed for other purposes",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Active / Inactive",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*CommitLimit - *./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 38
-          },
-          "id": 135,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Committed_AS - Amount of memory presently allocated on the system",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "CommitLimit - Amount of  memory currently available to be allocated on the system",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Committed",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 48
-          },
-          "id": 191,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Inactive_file - File-backed memory on inactive LRU list",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Active_file - File-backed memory on active LRU list",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs",
-              "refId": "D",
-              "step": 240
-            }
-          ],
-          "title": "Memory Active / Inactive Detail",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 48
-          },
-          "id": 130,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Writeback - Memory which is actively being written back to disk",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Dirty - Memory which is waiting to get written back to the disk",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Memory Writeback and Dirty",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"
-                },
-                "properties": [
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"
-                },
-                "properties": [
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 58
-          },
-          "id": 138,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages",
-              "refId": "D",
-              "step": 240
-            }
-          ],
-          "title": "Memory Shared and Mapped",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 58
-          },
-          "id": 131,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Slab",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 68
-          },
-          "id": 70,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "VmallocTotal - Total size of vmalloc memory area",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "VmallocUsed - Amount of vmalloc area which is used",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Memory Vmalloc",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 68
-          },
-          "id": 159,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Bounce - Memory used for block device bounce buffers",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Memory Bounce",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Inactive *./"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 78
-          },
-          "id": 129,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "AnonHugePages - Memory in anonymous huge pages",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "AnonPages - Memory in user pages not backed by files",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Anonymous",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 78
-          },
-          "id": 160,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Kernel / CPU",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "pages",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 88
-          },
-          "id": 140,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Memory HugePages Counter",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 88
-          },
-          "id": 71,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "HugePages - Total size of the pool of huge pages",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Hugepagesize - Huge Page size",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory HugePages Size",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 98
-          },
-          "id": 128,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "DirectMap1G - Amount of pages mapped as this size",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "DirectMap2M - Amount of pages mapped as this size",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "DirectMap4K - Amount of pages mapped as this size",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Memory DirectMap",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 98
-          },
-          "id": 137,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Unevictable and MLocked",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 108
-          },
-          "id": 132,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Memory NFS",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Memory Meminfo",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 22
-      },
-      "id": 267,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "pages out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*out/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 25
-          },
-          "id": 176,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Pagesin - Page in operations",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Pagesout - Page out operations",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Pages In / Out",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "pages out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*out/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 25
-          },
-          "id": 22,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Pswpin - Pages swapped in",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Pswpout - Pages swapped out",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Memory Pages Swap In / Out",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "faults",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Apps"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#629E51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A437C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#CFFAFF",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "RAM_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#806EB7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#2F575E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Unused"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Pgfault - Page major and minor fault operations"
-                },
-                "properties": [
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  },
-                  {
-                    "id": "custom.stacking",
-                    "value": {
-                      "group": false,
-                      "mode": "normal"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 35
-          },
-          "id": 175,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 350
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Pgfault - Page major and minor fault operations",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Pgmajfault - Major page fault operations",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])  - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Pgminfault - Minor page fault operations",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Memory Page Faults",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#99440A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Buffers"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#58140C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6D1F62",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Cached"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Committed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#508642",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Dirty"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Free"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#B7DBAB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Mapped"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PageTables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Page_Tables"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Slab_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Swap_Cache"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C15C17",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#511749",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total RAM + Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#052B51",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Total Swap"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "VmallocUsed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 35
-          },
-          "id": 307,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "oom killer invocations ",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "OOM Killer",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Memory Vmstat",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 23
-      },
-      "id": 293,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "seconds",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "s"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Variation*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 40
-          },
-          "id": 260,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Estimated error in seconds",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Time offset in between local system and reference clock",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Maximum error in seconds",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Time Synchronized Drift",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 40
-          },
-          "id": 291,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Phase-locked loop time adjust",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Time PLL Adjust",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Variation*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 50
-          },
-          "id": 168,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Local clock frequency adjustment",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Time Synchronized Status",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "seconds",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "s"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 50
-          },
-          "id": 294,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Seconds between clock ticks",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "International Atomic Time (TAI) offset",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Time Misc",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "System Timesync",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 24
-      },
-      "id": 312,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 27
-          },
-          "id": 62,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Processes blocked waiting for I/O to complete",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Processes in runnable state",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Processes Status",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 27
-          },
-          "id": 315,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ state }}",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Processes State",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "forks / sec",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 37
-          },
-          "id": 148,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Processes forks second",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Processes  Forks",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "decbytes"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Max.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 37
-          },
-          "id": 149,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Processes virtual memory size in bytes",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Maximum amount of virtual memory available in bytes",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Processes virtual memory size in bytes",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Maximum amount of virtual memory available in bytes",
-              "refId": "D",
-              "step": 240
-            }
-          ],
-          "title": "Processes Memory",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "PIDs limit"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F2495C",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 47
-          },
-          "id": 313,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Number of PIDs",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "PIDs limit",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "PIDs Number and Limit",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "seconds",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "s"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*waiting.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 47
-          },
-          "id": 305,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "CPU {{ cpu }} - seconds spent running a process",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Process schedule stats Running / Waiting",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Threads limit"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F2495C",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 57
-          },
-          "id": 314,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Allocated threads",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Threads limit",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Threads Number and Limit",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "System Processes",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 25
-      },
-      "id": 269,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 42
-          },
-          "id": 8,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Context switches",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "Interrupts",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Context Switches / Interrupts",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 42
-          },
-          "id": 7,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_load1{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 4,
-              "legendFormat": "Load 1m",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_load5{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 4,
-              "legendFormat": "Load 5m",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_load15{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 4,
-              "legendFormat": "Load 15m",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "System Load",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Critical*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Max*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 52
-          },
-          "id": 259,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ type }} - {{ info }}",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Interrupts Detail",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 52
-          },
-          "id": 306,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "CPU {{ cpu }}",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Schedule timeslices executed by each cpu",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 62
-          },
-          "id": 151,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Entropy available to random number generators",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Entropy",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "seconds",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "s"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 62
-          },
-          "id": 308,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Time spent",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "CPU time spent in user and system contexts",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Max*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 72
-          },
-          "id": 64,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Maximum open file descriptors",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Open file descriptors",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "File Descriptors",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "System Misc",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 26
-      },
-      "id": 304,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "temperature",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "celsius"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Critical*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Max*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 43
-          },
-          "id": 158,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ chip }} {{ sensor }} temp",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": true,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ chip }} {{ sensor }} Critical Alarm",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ chip }} {{ sensor }} Critical",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": true,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ chip }} {{ sensor }} Critical Historical",
-              "refId": "D",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": true,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ chip }} {{ sensor }} Max",
-              "refId": "E",
-              "step": 240
-            }
-          ],
-          "title": "Hardware temperature monitor",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Max*./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 43
-          },
-          "id": 300,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Current {{ name }} in {{ type }}",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Max {{ name }} in {{ type }}",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Throttle cooling device",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 53
-          },
-          "id": 302,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ power_supply }} online",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Power supply",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Hardware Misc",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 27
-      },
-      "id": 296,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 30
-          },
-          "id": 297,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{ name }} Connections",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Systemd Sockets",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Failed"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F2495C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Inactive"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FF9830",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Active"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#73BF69",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Deactivating"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FFCB7D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "Activating"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#C8F2C2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 30
-          },
-          "id": 298,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Activating",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Active",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Deactivating",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Failed",
-              "refId": "D",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Inactive",
-              "refId": "E",
-              "step": 240
-            }
-          ],
-          "title": "Systemd Units State",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Systemd",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 28
-      },
-      "id": 270,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "The number (after merges) of I/O requests completed per second for the device",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "IO read (-) / write (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "iops"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Read.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 31
-          },
-          "id": 9,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - Reads completed",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Writes completed",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Disk IOps Completed",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "The number of bytes read from or written to the device per second",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes read (-) / write (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "Bps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Read.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 31
-          },
-          "id": 33,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - Read bytes",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Written bytes",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Disk R/W Data",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "time. read (-) / write (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 30,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "s"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Read.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 41
-          },
-          "id": 37,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - Read wait time avg",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Write wait time avg",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Disk Average Wait Time",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "The average queue length of the requests that were issued to the device",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "aqu-sz",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "none"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 41
-          },
-          "id": 35,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}}",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Average Queue Size",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "The number of read and write requests merged per second that were queued to the device",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "I/Os",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "iops"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Read.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 51
-          },
-          "id": 133,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Read merged",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Write merged",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Disk R/W Merged",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially.  But for devices  serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "%util",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 30,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "percentunit"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 51
-          },
-          "id": 36,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - IO",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - discard",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Time Spent Doing I/Os",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "Outstanding req.",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "none"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 61
-          },
-          "id": 34,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}",
-              "interval": "",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - IO now",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Instantaneous Queue Size",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "IOs",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "iops"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EAB839",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#6ED0E0",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EF843C",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#584477",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda2_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BA43A9",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sda3_.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F4D598",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#0A50A1",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#BF1B00",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdb3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0752D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#962D82",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#614D93",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdc3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#9AC48A",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#65C5DB",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9934E",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#EA6460",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde1.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E0F9D7",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sdd2.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#FCEACA",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*sde3.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F9E2D2",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 61
-          },
-          "id": 301,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "intervalFactor": 4,
-              "legendFormat": "{{device}} - Discards completed",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Discards merged",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Disk IOps Discards completed / merged",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Storage Disk",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 29
-      },
-      "id": 271,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 46
-          },
-          "id": 43,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}} - Available",
-              "metric": "",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
-              "format": "time_series",
-              "hide": true,
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}} - Free",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
-              "format": "time_series",
-              "hide": true,
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}} - Size",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Filesystem space available",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "file nodes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 46
-          },
-          "id": 41,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}} - Free file nodes",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "File Nodes Free",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "files",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 56
-          },
-          "id": 28,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 4,
-              "legendFormat": "Max open files",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "Open files",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "File Descriptor",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "file Nodes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 56
-          },
-          "id": 219,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}} - File nodes total",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "File Nodes Size",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "max": 1,
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "/ ReadOnly"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 66
-          },
-          "id": 44,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}} - ReadOnly",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{mountpoint}} - Device error",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Filesystem in ReadOnly / Error",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Storage Filesystem",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 30
-      },
-      "id": 272,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "pps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "receive_packets_eth0"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "receive_packets_lo"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "transmit_packets_eth0"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#7EB26D",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "transmit_packets_lo"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#E24D42",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 31
-          },
-          "id": 60,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Transmit",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic by Packets",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "pps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 31
-          },
-          "id": 142,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive errors",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Rransmit errors",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Errors",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "pps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 41
-          },
-          "id": 143,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive drop",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Transmit drop",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Drop",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "pps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 41
-          },
-          "id": 141,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive compressed",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Transmit compressed",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Compressed",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "pps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 51
-          },
-          "id": 146,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive multicast",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Multicast",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "pps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 51
-          },
-          "id": 144,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive fifo",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Transmit fifo",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Fifo",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "pps"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 61
-          },
-          "id": 145,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Receive frame",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Frame",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 61
-          },
-          "id": 231,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Statistic transmit_carrier",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Carrier",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Trans.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 71
-          },
-          "id": 232,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{device}} - Transmit colls",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Network Traffic Colls",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "entries",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "NF conntrack limit"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 71
-          },
-          "id": 61,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "NF conntrack entries",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "NF conntrack limit",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "NF Contrack",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "Entries",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 81
-          },
-          "id": 230,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{ device }} - ARP entries",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "ARP Entries",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "decimals": 0,
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 81
-          },
-          "id": 288,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{ device }} - Bytes",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "MTU",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "decimals": 0,
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 91
-          },
-          "id": 280,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{ device }} - Speed",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Speed",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packets",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "decimals": 0,
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "none"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 91
-          },
-          "id": 289,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{ device }} -   Interface transmit queue length",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Queue Length",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "packetes drop (-) / process (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Dropped.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 101
-          },
-          "id": 290,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "CPU {{cpu}} - Processed",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "CPU {{cpu}} - Dropped",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Softnet Packets",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 101
-          },
-          "id": 310,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "CPU {{cpu}} - Squeezed",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Softnet Out of Quota",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 111
-          },
-          "id": 309,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{interface}} - Operational state UP",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "instant": false,
-              "legendFormat": "{{device}} - Physical link state",
-              "refId": "B"
-            }
-          ],
-          "title": "Network Operational Status",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Network Traffic",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 31
-      },
-      "id": 273,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 32
-          },
-          "id": 63,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "TCP_alloc - Allocated sockets",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "TCP_inuse - Tcp sockets currently in use",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": true,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "TCP_mem - Used memory for tcp",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "TCP_orphan - Orphan sockets",
-              "refId": "D",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "TCP_tw - Sockets waiting close",
-              "refId": "E",
-              "step": 240
-            }
-          ],
-          "title": "Sockstat TCP",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 32
-          },
-          "id": 124,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "UDP_inuse - Udp sockets currently in use",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "UDP_mem - Used memory for udp",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Sockstat UDP",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 42
-          },
-          "id": 125,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "FRAG_inuse - Frag sockets currently in use",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "RAW_inuse - Raw sockets currently in use",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "Sockstat FRAG / RAW",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "bytes",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "bytes"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 42
-          },
-          "id": 220,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "mem_bytes - TCP sockets in that state",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "mem_bytes - UDP sockets in that state",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "FRAG_memory - Used memory for frag",
-              "refId": "C"
-            }
-          ],
-          "title": "Sockstat Memory Size",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "sockets",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 52
-          },
-          "id": 126,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Sockets_used - Sockets currently in use",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Sockstat Used",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Network Sockstat",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 32
-      },
-      "id": 274,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "octets out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Out.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 33
-          },
-          "id": 221,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "InOctets - Received octets",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "OutOctets - Sent octets",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Netstat IP In / Out Octets",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "datagrams",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 33
-          },
-          "id": 81,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true,
-              "width": 300
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "Forwarding - IP forwarding",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Netstat IP Forwarding",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "messages out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Out.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 43
-          },
-          "id": 115,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "InMsgs -  Messages which the entity received. Note that this counter includes all those counted by icmpInErrors",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "ICMP In / Out",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "messages out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Out.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 43
-          },
-          "id": 50,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "ICMP Errors",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "datagrams out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Out.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Snd.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 53
-          },
-          "id": 55,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "InDatagrams - Datagrams received",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "OutDatagrams - Datagrams sent",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "UDP In / Out",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "datagrams",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 53
-          },
-          "id": 109,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application",
-              "refId": "C"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "RcvbufErrors - UDP buffer errors received",
-              "refId": "D",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "SndbufErrors - UDP buffer errors send",
-              "refId": "E",
-              "step": 240
-            }
-          ],
-          "title": "UDP Errors",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "datagrams out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Out.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Snd.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 63
-          },
-          "id": 299,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "TCP In / Out",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 63
-          },
-          "id": 104,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits",
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets",
-              "refId": "D"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)",
-              "refId": "E"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "interval": "",
-              "legendFormat": "OutRsts - Segments sent with RST flag",
-              "refId": "F"
-            }
-          ],
-          "title": "TCP Errors",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "connections",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*MaxConn *./"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#890F02",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.fillOpacity",
-                    "value": 0
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 73
-          },
-          "id": 85,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "TCP Connections",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter out (-) / in (+)",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*Sent.*/"
-                },
-                "properties": [
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 73
-          },
-          "id": 91,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "SyncookiesFailed - Invalid SYN cookies received",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "SyncookiesRecv - SYN cookies received",
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "SyncookiesSent - SYN cookies sent",
-              "refId": "C",
-              "step": 240
-            }
-          ],
-          "title": "TCP SynCookie",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "connections",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 83
-          },
-          "id": 82,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "TCP Direct Transition",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "Enable with --collector.tcpstat argument on node-exporter",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "connections",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 83
-          },
-          "id": 320,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "established - TCP sockets in established state",
-              "range": true,
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "fin_wait2 - TCP sockets in fin_wait2 state",
-              "range": true,
-              "refId": "B",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "listen - TCP sockets in listen state",
-              "range": true,
-              "refId": "C",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "editorMode": "code",
-              "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "time_wait - TCP sockets in time_wait state",
-              "range": true,
-              "refId": "D",
-              "step": 240
-            }
-          ],
-          "title": "TCP Stat",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Network Netstat",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "000000001"
-      },
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 33
-      },
-      "id": 279,
-      "panels": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "seconds",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "normal"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "s"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 0,
-            "y": 50
-          },
-          "id": 40,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{collector}} - Scrape duration",
-              "refId": "A",
-              "step": 240
-            }
-          ],
-          "title": "Node Exporter Scrape Time",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "P92AEBB27A9B79E22"
-          },
-          "description": "",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "counter",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 20,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "lineInterpolation": "linear",
-                "lineStyle": {
-                  "fill": "solid"
-                },
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "short"
-            },
-            "overrides": [
-              {
-                "matcher": {
-                  "id": "byRegexp",
-                  "options": "/.*error.*/"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "#F2495C",
-                      "mode": "fixed"
-                    }
-                  },
-                  {
-                    "id": "custom.transform",
-                    "value": "negative-Y"
-                  }
-                ]
-              }
-            ]
-          },
-          "gridPos": {
-            "h": 10,
-            "w": 12,
-            "x": 12,
-            "y": 50
-          },
-          "id": 157,
-          "links": [],
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max",
-                "min"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "none"
-            }
-          },
-          "pluginVersion": "9.2.0",
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{collector}} - Scrape success",
-              "refId": "A",
-              "step": 240
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "P92AEBB27A9B79E22"
-              },
-              "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{collector}} - Scrape textfile error (1 = true)",
-              "refId": "B",
-              "step": 240
-            }
-          ],
-          "title": "Node Exporter Scrape",
-          "type": "timeseries"
-        }
-      ],
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "000000001"
-          },
-          "refId": "A"
-        }
-      ],
-      "title": "Node Exporter",
-      "type": "row"
-    }
-  ],
-  "refresh": "5s",
-  "revision": 1,
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": [
-    "linux"
-  ],
-  "templating": {
-    "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "default",
-          "value": "default"
-        },
-        "hide": 0,
-        "includeAll": false,
-        "label": "datasource",
-        "multi": false,
-        "name": "DS_PROMETHEUS",
-        "options": [],
-        "query": "prometheus",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
-      {
-        "current": {
-          "selected": false,
-          "text": "node",
-          "value": "node"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "P92AEBB27A9B79E22"
-        },
-        "definition": "",
-        "hide": 0,
-        "includeAll": false,
-        "label": "Job",
-        "multi": false,
-        "name": "job",
-        "options": [],
-        "query": {
-          "query": "label_values(node_uname_info, job)",
-          "refId": "Prometheus-job-Variable-Query"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 1,
-        "tagValuesQuery": "",
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      },
-      {
-        "current": {
-          "selected": false,
-          "text": "node-exporter:9100",
-          "value": "node-exporter:9100"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "P92AEBB27A9B79E22"
-        },
-        "definition": "label_values(node_uname_info{job=\"$job\"}, instance)",
-        "hide": 0,
-        "includeAll": false,
-        "label": "Host",
-        "multi": false,
-        "name": "node",
-        "options": [],
-        "query": {
-          "query": "label_values(node_uname_info{job=\"$job\"}, instance)",
-          "refId": "Prometheus-node-Variable-Query"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 1,
-        "tagValuesQuery": "",
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      },
-      {
-        "current": {
-          "selected": false,
-          "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+",
-          "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"
-        },
-        "hide": 2,
-        "includeAll": false,
-        "multi": false,
-        "name": "diskdevices",
-        "options": [
-          {
-            "selected": true,
-            "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+",
-            "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"
-          }
-        ],
-        "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+",
-        "skipUrlSync": false,
-        "type": "custom"
-      }
-    ]
-  },
-  "time": {
-    "from": "now-30m",
-    "to": "now"
-  },
-  "timepicker": {
-    "refresh_intervals": [
-      "5s",
-      "10s",
-      "30s",
-      "1m",
-      "5m",
-      "15m",
-      "30m",
-      "1h",
-      "2h",
-      "1d"
-    ],
-    "time_options": [
-      "5m",
-      "15m",
-      "1h",
-      "6h",
-      "12h",
-      "24h",
-      "2d",
-      "7d",
-      "30d"
-    ]
-  },
-  "timezone": "browser",
-  "title": "Node Exporter Full",
-  "uid": "rYdddlPWk",
-  "version": 2,
-  "weekStart": ""
-}
\ No newline at end of file
+{"annotations":{"list":[{"$$hashKey":"object:1058","builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"gnetId":1860,"graphTooltip":1,"id":2,"links":[{"icon":"external link","tags":[],"targetBlank":true,"title":"GitHub","type":"link","url":"https://github.com/rfmoz/grafana-dashboards"},{"icon":"external link","tags":[],"targetBlank":true,"title":"Grafana","type":"link","url":"https://grafana.com/grafana/dashboards/1860"}],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":261,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Quick CPU / Mem / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":0,"y":1},"id":20,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100","hide":false,"instant":true,"intervalFactor":1,"legendFormat":"","range":false,"refId":"A","step":240}],"title":"CPU Busy","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together (5 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":3,"y":1},"id":155,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Sys Load (5m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together (15 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":6,"y":1},"id":19,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Sys Load (15m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Non available RAM memory","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":9,"y":1},"hideTimeOverride":false,"id":16,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100","format":"time_series","hide":true,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"B","step":240}],"title":"RAM Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Used Swap","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":10},{"color":"rgba(245, 54, 54, 0.9)","value":25}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":12,"y":1},"id":21,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"SWAP Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Used Root FS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":15,"y":1},"id":154,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))","format":"time_series","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Root FS Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total number of CPU cores","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":1},"id":14,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))","legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU Cores","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"System uptime","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":2,"w":4,"x":20,"y":1},"hideTimeOverride":true,"id":15,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Uptime","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total RootFS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":70},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":3},"id":23,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"RootFS Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total RAM","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":20,"y":3},"id":75,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"RAM Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total SWAP","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":22,"y":3},"id":18,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"SWAP Total","type":"stat"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":5},"id":263,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Basic CPU / Mem / Net / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic CPU info","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy System"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy User"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Other"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":6},"id":77,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy System","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy User","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Iowait","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy IRQs","range":true,"refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Other","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Idle","range":true,"refId":"F","step":240}],"title":"CPU Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic memory usage","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"SWAP Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Total"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]},{"matcher":{"id":"byName","options":"RAM Cache + Buffer"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Free"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"color","value":{"fixedColor":"#DEDAF7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":7,"w":12,"x":12,"y":6},"id":78,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Total","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Used","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Cache + Buffer","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Free","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","intervalFactor":1,"legendFormat":"SWAP Used","refId":"E","step":240}],"title":"Memory Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic network info per interface","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"Recv_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":13},"id":74,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"recv {{device}}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"trans {{device}} ","refId":"B","step":240}],"title":"Network Traffic Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Disk space used of all filesystems mounted","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":13},"id":152,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used Basic","type":"timeseries"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":20},"id":265,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Idle - Waiting for something to happen"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Iowait - Waiting for I/O to complete"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Irq - Servicing interrupts"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Nice - Niced processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Softirq - Servicing softirqs"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Steal - Time spent in other operating systems when running in a virtualized environment"},"properties":[{"id":"color","value":{"fixedColor":"#FCE2DE","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"System - Processes executing in kernel mode"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"User - Normal processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#5195CE","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":7},"id":3,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"System - Processes executing in kernel mode","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"User - Normal processes executing in user mode","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Nice - Niced processes executing in user mode","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Iowait - Waiting for I/O to complete","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Irq - Servicing interrupts","range":true,"refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Softirq - Servicing softirqs","range":true,"refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Steal - Time spent in other operating systems when running in a virtualized environment","range":true,"refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Idle - Waiting for something to happen","range":true,"refId":"J","step":240}],"title":"CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap - Swap memory usage"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused - Free memory unassigned"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Hardware Corrupted - *./"},"properties":[{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":7},"id":24,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Apps - Memory used by user-space applications","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"PageTables - Memory used to map between virtual and physical memory addresses","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Cache - Parked file data (file content) cache","refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Buffers - Block device (e.g. harddisk) cache","refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Unused - Free memory unassigned","refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Swap - Swap space used","refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working","refId":"I","step":240}],"title":"Memory Stack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bits out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":19},"id":84,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":12,"w":12,"x":12,"y":19},"id":156,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":31},"id":229,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*read*./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":31},"id":42,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully written bytes","refId":"B","step":240}],"title":"I/O Usage Read / Write","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":43},"id":127,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"I/O Utilization","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"bars","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":3,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":1,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/^Guest - /"},"properties":[{"id":"color","value":{"fixedColor":"#5195ce","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/^GuestNice - /"},"properties":[{"id":"color","value":{"fixedColor":"#c15c17","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":43},"id":319,"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"Guest - Time spent running a virtual CPU for a guest operating system","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"GuestNice - Time spent running a niced guest  (virtual CPU for guest operating system)","range":true,"refId":"B"}],"title":"CPU spent seconds in guests (VMs)","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"CPU / Memory / Net / Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":266,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":38},"id":136,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Inactive - Memory which has been less recently used.  It is more eligible to be reclaimed for other purposes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary","refId":"B","step":240}],"title":"Memory Active / Inactive","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*CommitLimit - *./"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":38},"id":135,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Committed_AS - Amount of memory presently allocated on the system","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"CommitLimit - Amount of  memory currently available to be allocated on the system","refId":"B","step":240}],"title":"Memory Committed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":48},"id":191,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_file - File-backed memory on inactive LRU list","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_file - File-backed memory on active LRU list","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs","refId":"D","step":240}],"title":"Memory Active / Inactive Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":48},"id":130,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Writeback - Memory which is actively being written back to disk","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"WritebackTmp - Memory used by FUSE for temporary writeback buffers","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Dirty - Memory which is waiting to get written back to the disk","refId":"C","step":240}],"title":"Memory Writeback and Dirty","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":58},"id":138,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Mapped - Used memory in mapped pages files which have been mapped, such as libraries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Shmem - Used shared memory (shared between several processes, thus including RAM disks)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages","refId":"D","step":240}],"title":"Memory Shared and Mapped","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":58},"id":131,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SReclaimable - Part of Slab, that might be reclaimed, such as caches","refId":"B","step":240}],"title":"Memory Slab","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":68},"id":70,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocChunk - Largest contiguous block of vmalloc area which is free","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocTotal - Total size of vmalloc memory area","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocUsed - Amount of vmalloc area which is used","refId":"C","step":240}],"title":"Memory Vmalloc","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":68},"id":159,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Bounce - Memory used for block device bounce buffers","refId":"A","step":240}],"title":"Memory Bounce","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Inactive *./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":78},"id":129,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonHugePages - Memory in anonymous huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonPages - Memory in user pages not backed by files","refId":"B","step":240}],"title":"Memory Anonymous","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":78},"id":160,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"KernelStack - Kernel memory stack. This is not reclaimable","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PerCPU - Per CPU memory allocated dynamically by loadable modules","refId":"B","step":240}],"title":"Memory Kernel / CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":88},"id":140,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Free - Huge pages in the pool that are not yet allocated","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages","refId":"C","step":240}],"title":"Memory HugePages Counter","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":88},"id":71,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages - Total size of the pool of huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Hugepagesize - Huge Page size","refId":"B","step":240}],"title":"Memory HugePages Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":98},"id":128,"links":[],"options":{"legend":{"calcs":["mean","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"DirectMap1G - Amount of pages mapped as this size","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap2M - Amount of pages mapped as this size","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap4K - Amount of pages mapped as this size","refId":"C","step":240}],"title":"Memory DirectMap","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":98},"id":137,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"MLocked - Size of pages locked to memory using the mlock() system call","refId":"B","step":240}],"title":"Memory Unevictable and MLocked","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":108},"id":132,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage","refId":"A","step":240}],"title":"Memory NFS","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Memory Meminfo","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":267,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":25},"id":176,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesin - Page in operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesout - Page out operations","refId":"B","step":240}],"title":"Memory Pages In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":25},"id":22,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpin - Pages swapped in","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpout - Pages swapped out","refId":"B","step":240}],"title":"Memory Pages Swap In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"faults","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Pgfault - Page major and minor fault operations"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":35},"id":175,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgfault - Page major and minor fault operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgmajfault - Major page fault operations","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])  - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgminfault - Minor page fault operations","refId":"C","step":240}],"title":"Memory Page Faults","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":35},"id":307,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"oom killer invocations ","refId":"A","step":240}],"title":"OOM Killer","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Memory Vmstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":23},"id":293,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":40},"id":260,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Estimated error in seconds","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Time offset in between local system and reference clock","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum error in seconds","refId":"C","step":240}],"title":"Time Synchronized Drift","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":40},"id":291,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Phase-locked loop time adjust","refId":"A","step":240}],"title":"Time PLL Adjust","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":168,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_sync_status{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Is clock synchronized to a reliable server (1 = yes, 0 = no)","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Local clock frequency adjustment","refId":"B","step":240}],"title":"Time Synchronized Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":294,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Seconds between clock ticks","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"International Atomic Time (TAI) offset","refId":"B","step":240}],"title":"Time Misc","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Timesync","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":24},"id":312,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":27},"id":62,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_procs_blocked{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes blocked waiting for I/O to complete","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_procs_running{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes in runnable state","refId":"B","step":240}],"title":"Processes Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":27},"id":315,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ state }}","refId":"A","step":240}],"title":"Processes State","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"forks / sec","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":37},"id":148,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Processes forks second","refId":"A","step":240}],"title":"Processes  Forks","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max.*/"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":37},"id":149,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"D","step":240}],"title":"Processes Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"PIDs limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":47},"id":313,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_pids{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Number of PIDs","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_max_processes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PIDs limit","refId":"B","step":240}],"title":"PIDs Number and Limit","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*waiting.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":47},"id":305,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent running a process","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent by processing waiting for this CPU","refId":"B","step":240}],"title":"Process schedule stats Running / Waiting","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Threads limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":57},"id":314,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Allocated threads","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_max_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Threads limit","refId":"B","step":240}],"title":"Threads Number and Limit","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Processes","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":25},"id":269,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":8,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Context switches","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Interrupts","refId":"B","step":240}],"title":"Context Switches / Interrupts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":7,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load1{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 1m","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load5{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 5m","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load15{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 15m","refId":"C","step":240}],"title":"System Load","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":259,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ type }} - {{ info }}","refId":"A","step":240}],"title":"Interrupts Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":52},"id":306,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }}","refId":"A","step":240}],"title":"Schedule timeslices executed by each cpu","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":62},"id":151,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_entropy_available_bits{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Entropy available to random number generators","refId":"A","step":240}],"title":"Entropy","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":62},"id":308,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Time spent","refId":"A","step":240}],"title":"CPU time spent in user and system contexts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":72},"id":64,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_max_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Maximum open file descriptors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_open_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Open file descriptors","refId":"B","step":240}],"title":"File Descriptors","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":26},"id":304,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"temperature","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"celsius"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":158,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} temp","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Alarm","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Historical","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Max","refId":"E","step":240}],"title":"Hardware temperature monitor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":300,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Current {{ name }} in {{ type }}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Max {{ name }} in {{ type }}","refId":"B","step":240}],"title":"Throttle cooling device","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":302,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_power_supply_online{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{ power_supply }} online","refId":"A","step":240}],"title":"Power supply","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Hardware Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":296,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":30},"id":297,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ name }} Connections","refId":"A","step":240}],"title":"Systemd Sockets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Failed"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#FF9830","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#73BF69","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Deactivating"},"properties":[{"id":"color","value":{"fixedColor":"#FFCB7D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Activating"},"properties":[{"id":"color","value":{"fixedColor":"#C8F2C2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":30},"id":298,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Activating","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Active","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Deactivating","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Failed","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Inactive","refId":"E","step":240}],"title":"Systemd Units State","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Systemd","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":28},"id":270,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number (after merges) of I/O requests completed per second for the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":9,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps Completed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of bytes read from or written to the device per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":33,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":4,"legendFormat":"{{device}} - Read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Written bytes","refId":"B","step":240}],"title":"Disk R/W Data","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"time. read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":41},"id":37,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":4,"legendFormat":"{{device}} - Read wait time avg","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}} - Write wait time avg","refId":"B","step":240}],"title":"Disk Average Wait Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The average queue length of the requests that were issued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"aqu-sz","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":41},"id":35,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"Average Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of read and write requests merged per second that were queued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"I/Os","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":51},"id":133,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Read merged","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Write merged","refId":"B","step":240}],"title":"Disk R/W Merged","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially.  But for devices  serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":51},"id":36,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - discard","refId":"B","step":240}],"title":"Time Spent Doing I/Os","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Outstanding req.","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":61},"id":34,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_disk_io_now{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO now","refId":"A","step":240}],"title":"Instantaneous Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IOs","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":61},"id":301,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - Discards completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Discards merged","refId":"B","step":240}],"title":"Disk IOps Discards completed / merged","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Storage Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":271,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":46},"id":43,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Available","metric":"","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Size","refId":"C","step":240}],"title":"Filesystem space available","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":46},"id":41,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free file nodes","refId":"A","step":240}],"title":"File Nodes Free","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"files","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":56},"id":28,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filefd_maximum{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Max open files","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filefd_allocated{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Open files","refId":"B","step":240}],"title":"File Descriptor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file Nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":56},"id":219,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - File nodes total","refId":"A","step":240}],"title":"File Nodes Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"/ ReadOnly"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":66},"id":44,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}} - ReadOnly","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{mountpoint}} - Device error","refId":"B","step":240}],"title":"Filesystem in ReadOnly / Error","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Storage Filesystem","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":272,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":60,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic by Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":142,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive errors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Rransmit errors","refId":"B","step":240}],"title":"Network Traffic Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":41},"id":143,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive drop","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit drop","refId":"B","step":240}],"title":"Network Traffic Drop","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":41},"id":141,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive compressed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit compressed","refId":"B","step":240}],"title":"Network Traffic Compressed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":51},"id":146,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive multicast","refId":"A","step":240}],"title":"Network Traffic Multicast","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":51},"id":144,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive fifo","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit fifo","refId":"B","step":240}],"title":"Network Traffic Fifo","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":61},"id":145,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Receive frame","refId":"A","step":240}],"title":"Network Traffic Frame","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":61},"id":231,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Statistic transmit_carrier","refId":"A","step":240}],"title":"Network Traffic Carrier","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":71},"id":232,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit colls","refId":"A","step":240}],"title":"Network Traffic Colls","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"NF conntrack limit"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":71},"id":61,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack entries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack limit","refId":"B","step":240}],"title":"NF Contrack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":81},"id":230,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_arp_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - ARP entries","refId":"A","step":240}],"title":"ARP Entries","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":81},"id":288,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Bytes","refId":"A","step":240}],"title":"MTU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":91},"id":280,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_speed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Speed","refId":"A","step":240}],"title":"Speed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":91},"id":289,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} -   Interface transmit queue length","refId":"A","step":240}],"title":"Queue Length","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packetes drop (-) / process (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Dropped.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":101},"id":290,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Processed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Dropped","refId":"B","step":240}],"title":"Softnet Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":101},"id":310,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Squeezed","refId":"A","step":240}],"title":"Softnet Out of Quota","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":111},"id":309,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{interface}} - Operational state UP","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_carrier{instance=\"$node\",job=\"$job\"}","format":"time_series","instant":false,"legendFormat":"{{device}} - Physical link state","refId":"B"}],"title":"Network Operational Status","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Traffic","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":273,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":32},"id":63,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_alloc - Allocated sockets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_inuse - Tcp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"TCP_mem - Used memory for tcp","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_orphan - Orphan sockets","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_tw - Sockets waiting close","refId":"E","step":240}],"title":"Sockstat TCP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":32},"id":124,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDPLITE_inuse - Udplite sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_inuse - Udp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_mem - Used memory for udp","refId":"C","step":240}],"title":"Sockstat UDP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":125,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"FRAG_inuse - Frag sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RAW_inuse - Raw sockets currently in use","refId":"C","step":240}],"title":"Sockstat FRAG / RAW","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":220,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - TCP sockets in that state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - UDP sockets in that state","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"FRAG_memory - Used memory for frag","refId":"C"}],"title":"Sockstat Memory Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"sockets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":126,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Sockets_used - Sockets currently in use","refId":"A","step":240}],"title":"Sockstat Used","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Sockstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":32},"id":274,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"octets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":33},"id":221,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InOctets - Received octets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"OutOctets - Sent octets","refId":"B","step":240}],"title":"Netstat IP In / Out Octets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":33},"id":81,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Forwarding - IP forwarding","refId":"A","step":240}],"title":"Netstat IP Forwarding","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":115,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InMsgs -  Messages which the entity received. Note that this counter includes all those counted by icmpInErrors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors","refId":"B","step":240}],"title":"ICMP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":50,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)","refId":"A","step":240}],"title":"ICMP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":55,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InDatagrams - Datagrams received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutDatagrams - Datagrams sent","refId":"B","step":240}],"title":"UDP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":53},"id":109,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - UDP Datagrams that could not be delivered to an application","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"NoPorts - UDP Datagrams received on a port with no listener","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrors Lite - UDPLite Datagrams that could not be delivered to an application","refId":"C"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RcvbufErrors - UDP buffer errors received","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"SndbufErrors - UDP buffer errors send","refId":"E","step":240}],"title":"UDP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":63},"id":299,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","instant":false,"interval":"","intervalFactor":1,"legendFormat":"InSegs - Segments received, including those received in error. This count includes segments received on currently established connections","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets","refId":"B","step":240}],"title":"TCP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":63},"id":104,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenOverflows - Times the listen queue of a socket overflowed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenDrops - SYNs to LISTEN sockets ignored","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets","refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrs - Segments received in error (e.g., bad TCP checksums)","refId":"E"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"OutRsts - Segments sent with RST flag","refId":"F"}],"title":"TCP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*MaxConn *./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":73},"id":85,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")","refId":"B","step":240}],"title":"TCP Connections","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Sent.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":73},"id":91,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesFailed - Invalid SYN cookies received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesRecv - SYN cookies received","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesSent - SYN cookies sent","refId":"C","step":240}],"title":"TCP SynCookie","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":83},"id":82,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state","refId":"B","step":240}],"title":"TCP Direct Transition","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Enable with --collector.tcpstat argument on node-exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":83},"id":320,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"established - TCP sockets in established state","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"fin_wait2 - TCP sockets in fin_wait2 state","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"listen - TCP sockets in listen state","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"time_wait - TCP sockets in time_wait state","range":true,"refId":"D","step":240}],"title":"TCP Stat","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Netstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":33},"id":279,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":40,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape duration","refId":"A","step":240}],"title":"Node Exporter Scrape Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineStyle":{"fill":"solid"},"lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*error.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":157,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_scrape_collector_success{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape success","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape textfile error (1 = true)","refId":"B","step":240}],"title":"Node Exporter Scrape","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Node Exporter","type":"row"}],"refresh":"5s","revision":1,"schemaVersion":38,"style":"dark","tags":["linux"],"templating":{"list":[{"current":{"selected":false,"text":"default","value":"default"},"hide":0,"includeAll":false,"label":"datasource","multi":false,"name":"DS_PROMETHEUS","options":[],"query":"prometheus","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":false,"text":"node","value":"node"},"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"definition":"","hide":0,"includeAll":false,"label":"Job","multi":false,"name":"job","options":[],"query":{"query":"label_values(node_uname_info, job)","refId":"Prometheus-job-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"node-exporter:9100","value":"node-exporter:9100"},"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"definition":"label_values(node_uname_info{job=\"$job\"}, instance)","hide":0,"includeAll":false,"label":"Host","multi":false,"name":"node","options":[],"query":{"query":"label_values(node_uname_info{job=\"$job\"}, instance)","refId":"Prometheus-node-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"},"hide":2,"includeAll":false,"multi":false,"name":"diskdevices","options":[{"selected":true,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"}],"query":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"Node Exporter Full","uid":"rYdddlPWk","version":2,"weekStart":""}
diff --git a/etc/grafana/dashboards/node-exporter-full.min.json b/etc/grafana/dashboards/node-exporter-full.min.json
deleted file mode 100644
index a782ea1d..00000000
--- a/etc/grafana/dashboards/node-exporter-full.min.json
+++ /dev/null
@@ -1 +0,0 @@
-{"annotations":{"list":[{"$$hashKey":"object:1058","builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"gnetId":1860,"graphTooltip":1,"id":2,"links":[{"icon":"external link","tags":[],"targetBlank":true,"title":"GitHub","type":"link","url":"https://github.com/rfmoz/grafana-dashboards"},{"icon":"external link","tags":[],"targetBlank":true,"title":"Grafana","type":"link","url":"https://grafana.com/grafana/dashboards/1860"}],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":261,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Quick CPU / Mem / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":0,"y":1},"id":20,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100","hide":false,"instant":true,"intervalFactor":1,"legendFormat":"","range":false,"refId":"A","step":240}],"title":"CPU Busy","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together (5 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":3,"y":1},"id":155,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Sys Load (5m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Busy state of all CPU cores together (15 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":6,"y":1},"id":19,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Sys Load (15m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Non available RAM memory","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":9,"y":1},"hideTimeOverride":false,"id":16,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100","format":"time_series","hide":true,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"B","step":240}],"title":"RAM Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Used Swap","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":10},{"color":"rgba(245, 54, 54, 0.9)","value":25}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":12,"y":1},"id":21,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"SWAP Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Used Root FS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":15,"y":1},"id":154,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))","format":"time_series","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Root FS Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total number of CPU cores","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":1},"id":14,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))","legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU Cores","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"System uptime","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":2,"w":4,"x":20,"y":1},"hideTimeOverride":true,"id":15,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"Uptime","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total RootFS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":70},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":3},"id":23,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"RootFS Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total RAM","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":20,"y":3},"id":75,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"RAM Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Total SWAP","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":22,"y":3},"id":18,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"10.1.1","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","exemplar":false,"expr":"node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}","instant":true,"intervalFactor":1,"range":false,"refId":"A","step":240}],"title":"SWAP Total","type":"stat"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":5},"id":263,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Basic CPU / Mem / Net / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic CPU info","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy System"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy User"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Other"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":6},"id":77,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy System","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy User","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Iowait","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy IRQs","range":true,"refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Other","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Idle","range":true,"refId":"F","step":240}],"title":"CPU Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic memory usage","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"SWAP Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Total"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]},{"matcher":{"id":"byName","options":"RAM Cache + Buffer"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Free"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"color","value":{"fixedColor":"#DEDAF7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":7,"w":12,"x":12,"y":6},"id":78,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Total","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Used","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Cache + Buffer","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Free","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","intervalFactor":1,"legendFormat":"SWAP Used","refId":"E","step":240}],"title":"Memory Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Basic network info per interface","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"Recv_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":13},"id":74,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"recv {{device}}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"trans {{device}} ","refId":"B","step":240}],"title":"Network Traffic Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Disk space used of all filesystems mounted","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":13},"id":152,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used Basic","type":"timeseries"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":20},"id":265,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Idle - Waiting for something to happen"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Iowait - Waiting for I/O to complete"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Irq - Servicing interrupts"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Nice - Niced processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Softirq - Servicing softirqs"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Steal - Time spent in other operating systems when running in a virtualized environment"},"properties":[{"id":"color","value":{"fixedColor":"#FCE2DE","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"System - Processes executing in kernel mode"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"User - Normal processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#5195CE","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":7},"id":3,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"System - Processes executing in kernel mode","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"User - Normal processes executing in user mode","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Nice - Niced processes executing in user mode","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Iowait - Waiting for I/O to complete","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Irq - Servicing interrupts","range":true,"refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Softirq - Servicing softirqs","range":true,"refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Steal - Time spent in other operating systems when running in a virtualized environment","range":true,"refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Idle - Waiting for something to happen","range":true,"refId":"J","step":240}],"title":"CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap - Swap memory usage"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused - Free memory unassigned"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Hardware Corrupted - *./"},"properties":[{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":7},"id":24,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Apps - Memory used by user-space applications","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"PageTables - Memory used to map between virtual and physical memory addresses","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Cache - Parked file data (file content) cache","refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Buffers - Block device (e.g. harddisk) cache","refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Unused - Free memory unassigned","refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Swap - Swap space used","refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working","refId":"I","step":240}],"title":"Memory Stack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bits out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":19},"id":84,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":12,"w":12,"x":12,"y":19},"id":156,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":31},"id":229,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*read*./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":31},"id":42,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully written bytes","refId":"B","step":240}],"title":"I/O Usage Read / Write","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":43},"id":127,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"I/O Utilization","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"bars","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":3,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":1,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/^Guest - /"},"properties":[{"id":"color","value":{"fixedColor":"#5195ce","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/^GuestNice - /"},"properties":[{"id":"color","value":{"fixedColor":"#c15c17","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":43},"id":319,"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"Guest - Time spent running a virtual CPU for a guest operating system","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"GuestNice - Time spent running a niced guest  (virtual CPU for guest operating system)","range":true,"refId":"B"}],"title":"CPU spent seconds in guests (VMs)","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"CPU / Memory / Net / Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":266,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":38},"id":136,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Inactive - Memory which has been less recently used.  It is more eligible to be reclaimed for other purposes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary","refId":"B","step":240}],"title":"Memory Active / Inactive","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*CommitLimit - *./"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":38},"id":135,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Committed_AS - Amount of memory presently allocated on the system","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"CommitLimit - Amount of  memory currently available to be allocated on the system","refId":"B","step":240}],"title":"Memory Committed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":48},"id":191,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_file - File-backed memory on inactive LRU list","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_file - File-backed memory on active LRU list","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs","refId":"D","step":240}],"title":"Memory Active / Inactive Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":48},"id":130,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Writeback - Memory which is actively being written back to disk","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"WritebackTmp - Memory used by FUSE for temporary writeback buffers","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Dirty - Memory which is waiting to get written back to the disk","refId":"C","step":240}],"title":"Memory Writeback and Dirty","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":58},"id":138,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Mapped - Used memory in mapped pages files which have been mapped, such as libraries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Shmem - Used shared memory (shared between several processes, thus including RAM disks)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated  with huge pages","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages","refId":"D","step":240}],"title":"Memory Shared and Mapped","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":58},"id":131,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SReclaimable - Part of Slab, that might be reclaimed, such as caches","refId":"B","step":240}],"title":"Memory Slab","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":68},"id":70,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocChunk - Largest contiguous block of vmalloc area which is free","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocTotal - Total size of vmalloc memory area","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocUsed - Amount of vmalloc area which is used","refId":"C","step":240}],"title":"Memory Vmalloc","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":68},"id":159,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Bounce - Memory used for block device bounce buffers","refId":"A","step":240}],"title":"Memory Bounce","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Inactive *./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":78},"id":129,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonHugePages - Memory in anonymous huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonPages - Memory in user pages not backed by files","refId":"B","step":240}],"title":"Memory Anonymous","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":78},"id":160,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"KernelStack - Kernel memory stack. This is not reclaimable","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PerCPU - Per CPU memory allocated dynamically by loadable modules","refId":"B","step":240}],"title":"Memory Kernel / CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":88},"id":140,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Free - Huge pages in the pool that are not yet allocated","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages","refId":"C","step":240}],"title":"Memory HugePages Counter","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":88},"id":71,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages - Total size of the pool of huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Hugepagesize - Huge Page size","refId":"B","step":240}],"title":"Memory HugePages Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":98},"id":128,"links":[],"options":{"legend":{"calcs":["mean","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"DirectMap1G - Amount of pages mapped as this size","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap2M - Amount of pages mapped as this size","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap4K - Amount of pages mapped as this size","refId":"C","step":240}],"title":"Memory DirectMap","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":98},"id":137,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"MLocked - Size of pages locked to memory using the mlock() system call","refId":"B","step":240}],"title":"Memory Unevictable and MLocked","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":108},"id":132,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage","refId":"A","step":240}],"title":"Memory NFS","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Memory Meminfo","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":267,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":25},"id":176,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesin - Page in operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesout - Page out operations","refId":"B","step":240}],"title":"Memory Pages In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":25},"id":22,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpin - Pages swapped in","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpout - Pages swapped out","refId":"B","step":240}],"title":"Memory Pages Swap In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"faults","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Pgfault - Page major and minor fault operations"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":35},"id":175,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgfault - Page major and minor fault operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgmajfault - Major page fault operations","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])  - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgminfault - Minor page fault operations","refId":"C","step":240}],"title":"Memory Page Faults","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":35},"id":307,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"oom killer invocations ","refId":"A","step":240}],"title":"OOM Killer","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Memory Vmstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":23},"id":293,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":40},"id":260,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Estimated error in seconds","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Time offset in between local system and reference clock","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum error in seconds","refId":"C","step":240}],"title":"Time Synchronized Drift","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":40},"id":291,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Phase-locked loop time adjust","refId":"A","step":240}],"title":"Time PLL Adjust","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":168,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_sync_status{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Is clock synchronized to a reliable server (1 = yes, 0 = no)","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Local clock frequency adjustment","refId":"B","step":240}],"title":"Time Synchronized Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":294,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Seconds between clock ticks","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"International Atomic Time (TAI) offset","refId":"B","step":240}],"title":"Time Misc","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Timesync","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":24},"id":312,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":27},"id":62,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_procs_blocked{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes blocked waiting for I/O to complete","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_procs_running{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes in runnable state","refId":"B","step":240}],"title":"Processes Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":27},"id":315,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ state }}","refId":"A","step":240}],"title":"Processes State","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"forks / sec","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":37},"id":148,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Processes forks second","refId":"A","step":240}],"title":"Processes  Forks","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max.*/"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":37},"id":149,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"D","step":240}],"title":"Processes Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"PIDs limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":47},"id":313,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_pids{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Number of PIDs","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_max_processes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PIDs limit","refId":"B","step":240}],"title":"PIDs Number and Limit","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*waiting.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":47},"id":305,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent running a process","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent by processing waiting for this CPU","refId":"B","step":240}],"title":"Process schedule stats Running / Waiting","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Threads limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":57},"id":314,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Allocated threads","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_processes_max_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Threads limit","refId":"B","step":240}],"title":"Threads Number and Limit","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Processes","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":25},"id":269,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":8,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Context switches","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Interrupts","refId":"B","step":240}],"title":"Context Switches / Interrupts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":7,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load1{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 1m","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load5{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 5m","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_load15{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 15m","refId":"C","step":240}],"title":"System Load","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":259,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ type }} - {{ info }}","refId":"A","step":240}],"title":"Interrupts Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":52},"id":306,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }}","refId":"A","step":240}],"title":"Schedule timeslices executed by each cpu","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":62},"id":151,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_entropy_available_bits{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Entropy available to random number generators","refId":"A","step":240}],"title":"Entropy","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":62},"id":308,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Time spent","refId":"A","step":240}],"title":"CPU time spent in user and system contexts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":72},"id":64,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_max_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Maximum open file descriptors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"process_open_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Open file descriptors","refId":"B","step":240}],"title":"File Descriptors","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"System Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":26},"id":304,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"temperature","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"celsius"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":158,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} temp","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Alarm","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Historical","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Max","refId":"E","step":240}],"title":"Hardware temperature monitor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":300,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Current {{ name }} in {{ type }}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Max {{ name }} in {{ type }}","refId":"B","step":240}],"title":"Throttle cooling device","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":302,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_power_supply_online{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{ power_supply }} online","refId":"A","step":240}],"title":"Power supply","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Hardware Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":296,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":30},"id":297,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ name }} Connections","refId":"A","step":240}],"title":"Systemd Sockets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Failed"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#FF9830","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#73BF69","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Deactivating"},"properties":[{"id":"color","value":{"fixedColor":"#FFCB7D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Activating"},"properties":[{"id":"color","value":{"fixedColor":"#C8F2C2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":30},"id":298,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Activating","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Active","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Deactivating","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Failed","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Inactive","refId":"E","step":240}],"title":"Systemd Units State","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Systemd","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":28},"id":270,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number (after merges) of I/O requests completed per second for the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":9,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps Completed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of bytes read from or written to the device per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":33,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":4,"legendFormat":"{{device}} - Read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Written bytes","refId":"B","step":240}],"title":"Disk R/W Data","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"time. read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":41},"id":37,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":4,"legendFormat":"{{device}} - Read wait time avg","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}} - Write wait time avg","refId":"B","step":240}],"title":"Disk Average Wait Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The average queue length of the requests that were issued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"aqu-sz","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":41},"id":35,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"Average Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of read and write requests merged per second that were queued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"I/Os","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":51},"id":133,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Read merged","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Write merged","refId":"B","step":240}],"title":"Disk R/W Merged","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially.  But for devices  serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":51},"id":36,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - discard","refId":"B","step":240}],"title":"Time Spent Doing I/Os","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Outstanding req.","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":61},"id":34,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_disk_io_now{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO now","refId":"A","step":240}],"title":"Instantaneous Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IOs","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":61},"id":301,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - Discards completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Discards merged","refId":"B","step":240}],"title":"Disk IOps Discards completed / merged","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Storage Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":271,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":46},"id":43,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Available","metric":"","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Size","refId":"C","step":240}],"title":"Filesystem space available","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":46},"id":41,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free file nodes","refId":"A","step":240}],"title":"File Nodes Free","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"files","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":56},"id":28,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filefd_maximum{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Max open files","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filefd_allocated{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Open files","refId":"B","step":240}],"title":"File Descriptor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file Nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":56},"id":219,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - File nodes total","refId":"A","step":240}],"title":"File Nodes Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"/ ReadOnly"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":66},"id":44,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}} - ReadOnly","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{mountpoint}} - Device error","refId":"B","step":240}],"title":"Filesystem in ReadOnly / Error","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Storage Filesystem","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":272,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":60,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic by Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":142,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive errors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Rransmit errors","refId":"B","step":240}],"title":"Network Traffic Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":41},"id":143,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive drop","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit drop","refId":"B","step":240}],"title":"Network Traffic Drop","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":41},"id":141,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive compressed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit compressed","refId":"B","step":240}],"title":"Network Traffic Compressed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":51},"id":146,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive multicast","refId":"A","step":240}],"title":"Network Traffic Multicast","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":51},"id":144,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive fifo","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit fifo","refId":"B","step":240}],"title":"Network Traffic Fifo","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":61},"id":145,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Receive frame","refId":"A","step":240}],"title":"Network Traffic Frame","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":61},"id":231,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Statistic transmit_carrier","refId":"A","step":240}],"title":"Network Traffic Carrier","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":71},"id":232,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit colls","refId":"A","step":240}],"title":"Network Traffic Colls","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"NF conntrack limit"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":71},"id":61,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack entries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack limit","refId":"B","step":240}],"title":"NF Contrack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":81},"id":230,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_arp_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - ARP entries","refId":"A","step":240}],"title":"ARP Entries","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":81},"id":288,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Bytes","refId":"A","step":240}],"title":"MTU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":91},"id":280,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_speed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Speed","refId":"A","step":240}],"title":"Speed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":91},"id":289,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} -   Interface transmit queue length","refId":"A","step":240}],"title":"Queue Length","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packetes drop (-) / process (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Dropped.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":101},"id":290,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Processed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Dropped","refId":"B","step":240}],"title":"Softnet Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":101},"id":310,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Squeezed","refId":"A","step":240}],"title":"Softnet Out of Quota","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":111},"id":309,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{interface}} - Operational state UP","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_network_carrier{instance=\"$node\",job=\"$job\"}","format":"time_series","instant":false,"legendFormat":"{{device}} - Physical link state","refId":"B"}],"title":"Network Operational Status","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Traffic","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":273,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":32},"id":63,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_alloc - Allocated sockets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_inuse - Tcp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"TCP_mem - Used memory for tcp","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_orphan - Orphan sockets","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_tw - Sockets waiting close","refId":"E","step":240}],"title":"Sockstat TCP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":32},"id":124,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDPLITE_inuse - Udplite sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_inuse - Udp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_mem - Used memory for udp","refId":"C","step":240}],"title":"Sockstat UDP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":125,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"FRAG_inuse - Frag sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RAW_inuse - Raw sockets currently in use","refId":"C","step":240}],"title":"Sockstat FRAG / RAW","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":220,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - TCP sockets in that state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - UDP sockets in that state","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"FRAG_memory - Used memory for frag","refId":"C"}],"title":"Sockstat Memory Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"sockets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":126,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Sockets_used - Sockets currently in use","refId":"A","step":240}],"title":"Sockstat Used","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Sockstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":32},"id":274,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"octets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":33},"id":221,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InOctets - Received octets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"OutOctets - Sent octets","refId":"B","step":240}],"title":"Netstat IP In / Out Octets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":33},"id":81,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Forwarding - IP forwarding","refId":"A","step":240}],"title":"Netstat IP Forwarding","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":115,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InMsgs -  Messages which the entity received. Note that this counter includes all those counted by icmpInErrors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors","refId":"B","step":240}],"title":"ICMP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":50,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)","refId":"A","step":240}],"title":"ICMP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":55,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InDatagrams - Datagrams received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutDatagrams - Datagrams sent","refId":"B","step":240}],"title":"UDP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":53},"id":109,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - UDP Datagrams that could not be delivered to an application","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"NoPorts - UDP Datagrams received on a port with no listener","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrors Lite - UDPLite Datagrams that could not be delivered to an application","refId":"C"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RcvbufErrors - UDP buffer errors received","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"SndbufErrors - UDP buffer errors send","refId":"E","step":240}],"title":"UDP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":63},"id":299,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","instant":false,"interval":"","intervalFactor":1,"legendFormat":"InSegs - Segments received, including those received in error. This count includes segments received on currently established connections","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets","refId":"B","step":240}],"title":"TCP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":63},"id":104,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenOverflows - Times the listen queue of a socket overflowed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenDrops - SYNs to LISTEN sockets ignored","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets","refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrs - Segments received in error (e.g., bad TCP checksums)","refId":"E"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"OutRsts - Segments sent with RST flag","refId":"F"}],"title":"TCP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*MaxConn *./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":73},"id":85,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")","refId":"B","step":240}],"title":"TCP Connections","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Sent.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":73},"id":91,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesFailed - Invalid SYN cookies received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesRecv - SYN cookies received","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesSent - SYN cookies sent","refId":"C","step":240}],"title":"TCP SynCookie","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":83},"id":82,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state","refId":"B","step":240}],"title":"TCP Direct Transition","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"Enable with --collector.tcpstat argument on node-exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":83},"id":320,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"established - TCP sockets in established state","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"fin_wait2 - TCP sockets in fin_wait2 state","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"listen - TCP sockets in listen state","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"time_wait - TCP sockets in time_wait state","range":true,"refId":"D","step":240}],"title":"TCP Stat","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Network Netstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":33},"id":279,"panels":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":40,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape duration","refId":"A","step":240}],"title":"Node Exporter Scrape Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineStyle":{"fill":"solid"},"lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*error.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":157,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_scrape_collector_success{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape success","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"expr":"node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape textfile error (1 = true)","refId":"B","step":240}],"title":"Node Exporter Scrape","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Node Exporter","type":"row"}],"refresh":"5s","revision":1,"schemaVersion":38,"style":"dark","tags":["linux"],"templating":{"list":[{"current":{"selected":false,"text":"default","value":"default"},"hide":0,"includeAll":false,"label":"datasource","multi":false,"name":"DS_PROMETHEUS","options":[],"query":"prometheus","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":false,"text":"node","value":"node"},"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"definition":"","hide":0,"includeAll":false,"label":"Job","multi":false,"name":"job","options":[],"query":{"query":"label_values(node_uname_info, job)","refId":"Prometheus-job-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"node-exporter:9100","value":"node-exporter:9100"},"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"definition":"label_values(node_uname_info{job=\"$job\"}, instance)","hide":0,"includeAll":false,"label":"Host","multi":false,"name":"node","options":[],"query":{"query":"label_values(node_uname_info{job=\"$job\"}, instance)","refId":"Prometheus-node-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"},"hide":2,"includeAll":false,"multi":false,"name":"diskdevices","options":[{"selected":true,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"}],"query":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"Node Exporter Full","uid":"rYdddlPWk","version":2,"weekStart":""}
diff --git a/scripts/minimize-dashboards.sh b/scripts/minimize-dashboards.sh
index e44face7..e2d6b973 100755
--- a/scripts/minimize-dashboards.sh
+++ b/scripts/minimize-dashboards.sh
@@ -7,7 +7,6 @@ set -e
 
 DIR="etc/grafana/dashboards"
 
-
 for dashboard in "${DIR}"/*.json; do
     if [[ "$dashboard" == *.min.json ]]; then
         continue
@@ -16,6 +15,11 @@ for dashboard in "${DIR}"/*.json; do
     jq -c < "$dashboard" > "${DIR}/${name}.min.json"
 done
 
+for dashboard in "${DIR}"/*.min.json; do
+    name=$(basename "$dashboard" .min.json)
+    mv "$dashboard" "${DIR}/${name}.json"
+done
+
 if [ "$1" == "--check" ] ; then
  if ! git diff --exit-code; then
     echo "Please run minimize-dashboards.sh and commit after editing the grafana dashboards."

From be5d3b84bc00169be00081f765219e26852478fb Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 27 Oct 2023 10:34:28 +0800
Subject: [PATCH 154/261] refactor: refine write model to reduce tail latency
 (#184)

* refactor: refine write model to reduce tail latency

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused output

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: bring back reclaimer

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix total bytes metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update grafana

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: replace blocks with capacity in config

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update codecov.yml

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update codecov.yml

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update codecov.yml

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 codecov.yml                            |  18 ++
 etc/grafana/dashboards/foyer.json      |   2 +-
 foyer-common/src/continuum.rs          |   5 +-
 foyer-storage-bench/src/main.rs        |   5 +
 foyer-storage/src/admission/mod.rs     |   2 +-
 foyer-storage/src/buffer.rs            | 340 +++++++++++++++++++++++++
 foyer-storage/src/catalog.rs           |  47 ++--
 foyer-storage/src/error.rs             |  10 +-
 foyer-storage/src/flusher_v2.rs        | 203 +++++++++++++++
 foyer-storage/src/generic.rs           | 202 ++++++++++-----
 foyer-storage/src/lazy.rs              |   2 +
 foyer-storage/src/lib.rs               |   2 +
 foyer-storage/src/reclaimer.rs         |   2 +-
 foyer-storage/src/reinsertion/exist.rs |   8 +-
 foyer-storage/src/ring.rs              | 203 +++++++++------
 foyer-storage/tests/storage_test.rs    |   4 +
 16 files changed, 876 insertions(+), 179 deletions(-)
 create mode 100644 foyer-storage/src/buffer.rs
 create mode 100644 foyer-storage/src/flusher_v2.rs

diff --git a/codecov.yml b/codecov.yml
index 0d75e0fd..873b19f0 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -2,5 +2,23 @@
 # Reference: https://docs.codecov.com/docs/codecovyml-reference
 # Tips. You may run following command to validate before committing any changes
 #   curl --data-binary @codecov.yml https://codecov.io/validate
+coverage:
+  precision: 2
+  round: down
+  range: "70..100"
+  status:
+    project:
+      default:
+        target: auto
+        threshold: "1%"
+        only_pulls: true
+    patch:
+      default:
+        informational: true
+        only_pulls: true
+    changes:
+      default:
+        informational: true
+        only_pulls: true
 ignore:
   - "foyer-storage-bench"
diff --git a/etc/grafana/dashboards/foyer.json b/etc/grafana/dashboards/foyer.json
index 50d5f8eb..de5308ef 100644
--- a/etc/grafana/dashboards/foyer.json
+++ b/etc/grafana/dashboards/foyer.json
@@ -1 +1 @@
-{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":2,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":2,"weekStart":""}
+{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Write Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":10,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Read Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":9,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":25},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":33},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""}
diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
index 2312ec5c..15e47a8e 100644
--- a/foyer-common/src/continuum.rs
+++ b/foyer-common/src/continuum.rs
@@ -67,7 +67,7 @@ macro_rules! def_continuum {
                 /// Submit a range, may advance continuum till the given range.
                 ///
                 /// Return `true` if advanced, else `false`.
-                pub fn submit_advance(&self, range: Range<$uint>) -> bool{
+                pub fn submit_advance(&self, range: Range<$uint>) -> bool {
                     debug_assert!(range.start < range.end);
 
                     let continuum = self.continuum.load(Ordering::Acquire);
@@ -166,8 +166,7 @@ macro_rules! def_continuum {
                         continuum = next;
                     }
 
-                    #[cfg(test)]
-                    assert_eq!(start, self.continuum.load(Ordering::Acquire));
+                    debug_assert_eq!(start, self.continuum.load(Ordering::Acquire));
 
                     // modify continuum exclusively and unlock
                     self.continuum.store(continuum, Ordering::Release);
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 0f02526f..94e57396 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -111,6 +111,10 @@ pub struct Args {
     #[arg(long, default_value_t = 1024)]
     buffer_pool_size: usize,
 
+    /// (MiB)
+    #[arg(long, default_value_t = 1024)]
+    ring_buffer_capacity: usize,
+
     #[arg(long, default_value_t = 4)]
     flushers: usize,
 
@@ -552,6 +556,7 @@ async fn main() {
         eviction_config,
         device_config,
         allocator_bits: args.allocator_bits,
+        ring_buffer_capacity: args.ring_buffer_capacity * 1024 * 1024,
         catalog_bits: args.catalog_bits,
         admissions,
         reinsertions,
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index e6bae0d5..a5329686 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -23,7 +23,7 @@ pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, indices: &Arc<Catalog<Self::Key>>) {}
+    fn init(&self, catalog: &Arc<Catalog<Self::Key>>) {}
 
     fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
 
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
new file mode 100644
index 00000000..bd6132e6
--- /dev/null
+++ b/foyer-storage/src/buffer.rs
@@ -0,0 +1,340 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use foyer_common::bits::{align_up, is_aligned};
+
+use crate::{
+    device::{error::DeviceError, Device},
+    flusher_v2::Entry,
+    region::{RegionHeader, RegionId, REGION_MAGIC},
+};
+
+#[derive(thiserror::Error, Debug)]
+pub enum BufferError {
+    #[error(transparent)]
+    Device(#[from] DeviceError),
+    #[error("")]
+    NotEnough { entry: Entry },
+}
+
+pub type BufferResult<T> = std::result::Result<T, BufferError>;
+
+#[derive(Debug)]
+pub struct PositionedEntry {
+    pub entry: Entry,
+    pub region: RegionId,
+    pub offset: usize,
+}
+
+#[derive(Debug)]
+pub struct FlushBuffer<D>
+where
+    D: Device,
+{
+    /// io buffer
+    ///
+    /// # Safety
+    ///
+    /// `buffer` should always be `Some`. The usage of `Option` is for temporarily taking ownership.
+    buffer: Option<Vec<u8, D::IoBufferAllocator>>,
+
+    /// current writing region
+    region: Option<RegionId>,
+
+    /// current buffer offset of current writing region
+    offset: usize,
+
+    /// entries in io buffer waiting for flush
+    entries: Vec<PositionedEntry>,
+
+    // underlying device
+    device: D,
+}
+
+impl<D> FlushBuffer<D>
+where
+    D: Device,
+{
+    pub fn new(device: D) -> Self {
+        let buffer = Some(device.io_buffer(0, device.io_size()));
+        Self {
+            buffer,
+            region: None,
+            offset: 0,
+            entries: vec![],
+            device,
+        }
+    }
+
+    pub fn region(&self) -> Option<RegionId> {
+        self.region
+    }
+
+    pub fn remaining(&self) -> usize {
+        if self.region.is_none() {
+            return 0;
+        }
+        self.device.region_size() - self.offset - self.buffer.as_ref().unwrap().len()
+    }
+
+    /// Flush io buffer if necessary, and reset io buffer to a new region.
+    ///
+    /// Returns fully flushed entries.
+    pub async fn rotate(&mut self, region: RegionId) -> BufferResult<Vec<PositionedEntry>> {
+        let entries = self.flush().await?;
+        debug_assert!(self.buffer.as_ref().unwrap().is_empty());
+        self.region = Some(region);
+        self.offset = 0;
+
+        // write region header
+        let buffer = self.buffer.as_mut().unwrap();
+        unsafe { buffer.set_len(self.device.align()) };
+        let header = RegionHeader {
+            magic: REGION_MAGIC,
+        };
+        header.write(buffer);
+
+        Ok(entries)
+    }
+
+    /// Flush io buffer and move the io buffer to the next position.
+    ///
+    /// The io buffer will be cleared after flush.
+    ///
+    /// Returns fully flushed entries.
+    pub async fn flush(&mut self) -> BufferResult<Vec<PositionedEntry>> {
+        let Some(region) = self.region else {
+            debug_assert!(self.entries.is_empty());
+            return Ok(vec![]);
+        };
+
+        // align io buffer
+        let mut buffer = self.buffer.take().unwrap();
+        let len = align_up(self.device.align(), buffer.len());
+        debug_assert!(len <= buffer.capacity());
+        unsafe { buffer.set_len(len) };
+        debug_assert!(self.offset + buffer.len() <= self.device.region_size());
+
+        // flush and clear buffer
+        let (res, mut buffer) = self
+            .device
+            .write(buffer, .., region, self.offset as u64)
+            .await;
+        buffer.clear();
+        self.buffer = Some(buffer);
+        res?;
+
+        // advance io buffer
+        self.offset += self.device.io_size();
+        if self.offset == self.device.region_size() {
+            self.region = None;
+        }
+
+        let mut entries = vec![];
+        std::mem::swap(&mut self.entries, &mut entries);
+        Ok(entries)
+    }
+
+    /// Write entry to io buffer.
+    ///
+    /// The io buffer may be flushed if needed.
+    ///
+    /// Returns fully flushed entries if there is enough space in the current region.
+    /// Otherwise, returns `NotEnough` error with the given `entry`.
+    pub async fn write(&mut self, entry: Entry) -> BufferResult<Vec<PositionedEntry>> {
+        // check region remaining size
+        let padding = align_up(self.device.align(), entry.view.len()) - entry.view.len();
+        if self.remaining() < entry.view.len() + padding {
+            return Err(BufferError::NotEnough { entry });
+        }
+
+        let region = self.region.unwrap();
+        let offset = self.offset + self.buffer.as_ref().unwrap().len();
+
+        let mut entries = vec![];
+        let mut written = 0;
+
+        // write view
+        let mut flushed = true;
+        while written < entry.view.len() {
+            flushed = false;
+            let buffer = self.buffer.as_mut().unwrap();
+            let bytes = std::cmp::min(
+                entry.view.len() - written,
+                self.device.io_size() - buffer.len(),
+            );
+            std::io::copy(&mut &entry.view[written..written + bytes], buffer)
+                .map_err(DeviceError::from)?;
+            written += bytes;
+
+            if buffer.len() == self.device.io_size() {
+                entries.append(&mut self.flush().await?);
+                flushed = true;
+            }
+        }
+
+        // write padding
+        let buffer = self.buffer.as_mut().unwrap();
+        debug_assert!(self.device.io_size() - buffer.len() >= padding);
+        unsafe { buffer.set_len(buffer.len() + padding) };
+        debug_assert!(is_aligned(self.device.align(), buffer.len()));
+        if buffer.len() == self.device.io_size() {
+            entries.append(&mut self.flush().await?);
+            flushed = true;
+        }
+
+        let entry = PositionedEntry {
+            entry,
+            region,
+            offset,
+        };
+        if flushed {
+            entries.push(entry);
+        } else {
+            self.entries.push(entry);
+        }
+
+        Ok(entries)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use bytes::BufMut;
+    use tempfile::tempdir;
+
+    use crate::{
+        device::fs::{FsDevice, FsDeviceConfig},
+        ring::{RingBuffer, View},
+    };
+
+    use super::*;
+
+    fn ent(view: View) -> Entry {
+        Entry {
+            key: Arc::new(()),
+            view,
+            key_len: 0,
+            value_len: 0,
+            sequence: 0,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_flush_buffer() {
+        let tempdir = tempdir().unwrap();
+
+        let ring = Arc::new(RingBuffer::new(4096, 16 * 1024 * 1024));
+        let device = FsDevice::open(FsDeviceConfig {
+            dir: tempdir.path().into(),
+            capacity: 256 * 1024,     // 256 KiB
+            file_capacity: 64 * 1024, // 64 KiB
+            align: 4 * 1024,          // 4 KiB
+            io_size: 16 * 1024,       // 16 KiB
+        })
+        .await
+        .unwrap();
+
+        let mut buffer = FlushBuffer::new(device.clone());
+        assert_eq!(buffer.region(), None);
+
+        {
+            let view = {
+                let mut view = ring.allocate(5 * 1024 - 128, 0).await; // ~ 6 KiB
+                (&mut view[..]).put_slice(&[b'x'; 5 * 1024 - 128]);
+                view.shrink_to(5 * 1024 - 128); // ~ 5 KiB
+                view.freeze()
+            };
+            let entry = ent(view);
+            assert_eq!(ring.continuum(), 0);
+
+            let res = buffer.write(entry).await;
+            let entry = match res {
+                Err(BufferError::NotEnough { entry }) => entry,
+                _ => panic!("should be not enough error"),
+            };
+
+            let entries = buffer.rotate(0).await.unwrap();
+            assert!(entries.is_empty());
+
+            let entries = buffer.write(entry.clone()).await.unwrap();
+            assert!(entries.is_empty());
+            let entries = buffer.write(entry.clone()).await.unwrap();
+            assert_eq!(entries.len(), 1);
+            assert_eq!(entries[0].offset, 4 * 1024);
+
+            let entries = buffer.flush().await.unwrap();
+            assert_eq!(entries.len(), 1);
+            assert_eq!(entries[0].offset, 12 * 1024);
+
+            let buf = device.io_buffer(64 * 1024, 64 * 1024);
+            let (res, buf) = device.read(buf, .., 0, 0).await;
+            res.unwrap();
+            assert_eq!(&buf[4 * 1024..9 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
+            assert_eq!(&buf[12 * 1024..17 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
+
+            assert!(buffer.entries.is_empty());
+        }
+
+        ring.advance();
+        assert_eq!(ring.continuum(), 8 * 1024);
+
+        {
+            let view = {
+                let mut view = ring.allocate(55 * 1024 - 128, 1).await; // ~ 55 KiB
+                (&mut view[..]).put_slice(&[b'x'; 54 * 1024 - 128]);
+                view.shrink_to(54 * 1024 - 128); // ~ 54 KiB
+                view.freeze()
+            };
+            let entry = ent(view);
+
+            let res = buffer.write(entry).await;
+            let entry = match res {
+                Err(BufferError::NotEnough { entry }) => entry,
+                _ => panic!("should be not enough error"),
+            };
+
+            let entries = buffer.rotate(1).await.unwrap();
+            assert!(entries.is_empty());
+
+            let entries = buffer.write(entry).await.unwrap();
+            assert!(entries.is_empty());
+
+            let view = {
+                let mut view = ring.allocate(3 * 1024 - 128, 2).await; // ~ 3 KiB
+                (&mut view[..]).put_slice(&[b'x'; 3 * 1024 - 128]);
+                view.freeze()
+            };
+            let entry = ent(view);
+
+            let entries = buffer.write(entry).await.unwrap();
+            assert_eq!(entries.len(), 2);
+            assert_eq!(entries[0].offset, 4 * 1024);
+            assert_eq!(entries[1].offset, 60 * 1024);
+
+            let buf = device.io_buffer(64 * 1024, 64 * 1024);
+            let (res, buf) = device.read(buf, .., 1, 0).await;
+            res.unwrap();
+            assert_eq!(&buf[4 * 1024..58 * 1024 - 128], &[b'x'; 54 * 1024 - 128]);
+            assert_eq!(&buf[60 * 1024..63 * 1024 - 128], &[b'x'; 3 * 1024 - 128]);
+
+            assert!(buffer.entries.is_empty());
+        }
+
+        ring.advance();
+        assert_eq!(ring.continuum(), 68 * 1024);
+    }
+}
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 2f8b99b7..17b5a3f5 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -23,13 +23,19 @@ use itertools::Itertools;
 use parking_lot::{Mutex, RwLock};
 use twox_hash::XxHash64;
 
-use crate::region::{RegionId, Version};
+use crate::{
+    device::BufferAllocator,
+    region::{RegionId, Version},
+    ring::View,
+};
 
 pub type Sequence = u64;
 
 #[derive(Debug, Clone)]
 pub enum Index {
-    RingBuffer {},
+    RingBuffer {
+        view: View,
+    },
     Region {
         region: RegionId,
         version: Version,
@@ -41,7 +47,7 @@ pub enum Index {
 }
 
 #[derive(Debug, Clone)]
-pub struct IndexInfo {
+pub struct Item {
     pub sequence: Sequence,
     pub index: Index,
 }
@@ -55,7 +61,7 @@ where
     bits: usize,
 
     /// Sharded by key hash.
-    infos: Vec<RwLock<BTreeMap<Arc<K>, IndexInfo>>>,
+    items: Vec<RwLock<BTreeMap<Arc<K>, Item>>>,
 
     /// Sharded by region id.
     regions: Vec<Mutex<BTreeMap<Arc<K>, u64>>>,
@@ -74,62 +80,61 @@ where
             .collect_vec();
         Self {
             bits,
-            infos,
+            items: infos,
             regions,
         }
     }
 
-    pub fn insert(&self, key: K, info: IndexInfo) {
+    pub fn insert(&self, key: Arc<K>, item: Item) {
         // TODO(MrCroxx): compare sequence.
-        let key = Arc::new(key);
 
-        if let Index::Region { region, .. } = info.index {
+        if let Index::Region { region, .. } = item.index {
             self.regions[region as usize]
                 .lock()
-                .insert(key.clone(), info.sequence);
+                .insert(key.clone(), item.sequence);
         };
 
         let shard = self.shard(&key);
         // TODO(MrCroxx): handle old key?
-        let _ = self.infos[shard].write().insert(key.clone(), info);
+        let _ = self.items[shard].write().insert(key.clone(), item);
     }
 
-    pub fn lookup(&self, key: &K) -> Option<IndexInfo> {
+    pub fn lookup(&self, key: &K) -> Option<Item> {
         let shard = self.shard(key);
-        self.infos[shard].read().get(key).cloned()
+        self.items[shard].read().get(key).cloned()
     }
 
-    pub fn remove(&self, key: &K) -> Option<IndexInfo> {
+    pub fn remove(&self, key: &K) -> Option<Item> {
         let shard = self.shard(key);
-        let info: Option<IndexInfo> = self.infos[shard].write().remove(key);
+        let info: Option<Item> = self.items[shard].write().remove(key);
         if let Some(info) = &info && let Index::Region { region,..} = info.index {
             self.regions[region as usize].lock().remove(key);
         }
         info
     }
 
-    pub fn take_region(&self, region: &RegionId) -> Vec<IndexInfo> {
+    pub fn take_region(&self, region: &RegionId) -> Vec<(Arc<K>, Item)> {
         let mut keys = BTreeMap::new();
         std::mem::swap(&mut *self.regions[*region as usize].lock(), &mut keys);
 
-        let mut infos = Vec::with_capacity(keys.len());
+        let mut items = Vec::with_capacity(keys.len());
         for (key, sequence) in keys {
             let shard = self.shard(&key);
-            match self.infos[shard].write().entry(key.clone()) {
+            match self.items[shard].write().entry(key.clone()) {
                 Entry::Vacant(_) => continue,
                 Entry::Occupied(o) => {
                     if o.get().sequence == sequence {
-                        let info = o.remove();
-                        infos.push(info);
+                        let item = o.remove();
+                        items.push((key.clone(), item));
                     }
                 }
             };
         }
-        infos
+        items
     }
 
     pub fn clear(&self) {
-        for shard in self.infos.iter() {
+        for shard in self.items.iter() {
             shard.write().clear();
         }
         for region in self.regions.iter() {
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index a48ac3b3..b484dda1 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::device::error::DeviceError;
+use crate::{buffer::BufferError, device::error::DeviceError};
 
 #[derive(thiserror::Error, Debug)]
 #[error("{0}")]
@@ -30,6 +30,8 @@ struct ErrorInner {
 pub enum ErrorKind {
     #[error("device error: {0}")]
     Device(#[from] DeviceError),
+    #[error("buffer error: {0}")]
+    Buffer(#[from] BufferError),
     #[error("other error: {0}")]
     Other(#[from] anyhow::Error),
 }
@@ -46,6 +48,12 @@ impl From<DeviceError> for Error {
     }
 }
 
+impl From<BufferError> for Error {
+    fn from(value: BufferError) -> Self {
+        value.into()
+    }
+}
+
 impl From<anyhow::Error> for Error {
     fn from(value: anyhow::Error) -> Self {
         value.into()
diff --git a/foyer-storage/src/flusher_v2.rs b/foyer-storage/src/flusher_v2.rs
new file mode 100644
index 00000000..14ea0888
--- /dev/null
+++ b/foyer-storage/src/flusher_v2.rs
@@ -0,0 +1,203 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use foyer_common::{code::Key, rate::RateLimiter};
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use std::{any::Any, sync::Arc};
+use tokio::sync::{broadcast, mpsc};
+use tracing::Instrument;
+
+use crate::{
+    buffer::{BufferError, FlushBuffer, PositionedEntry},
+    catalog::{Catalog, Index, Item, Sequence},
+    device::Device,
+    error::{Error, Result},
+    metrics::Metrics,
+    region_manager::{RegionEpItemAdapter, RegionManager},
+    ring::View,
+};
+
+#[derive(Debug)]
+pub struct Entry {
+    /// # Safety
+    ///
+    /// `key` must be `Arc<K> where K = Flusher<K>`.
+    ///
+    /// Use `dyn Any` here to avoid contagious generic type.
+    pub key: Arc<dyn Any + Send + Sync>,
+    pub key_len: usize,
+    pub value_len: usize,
+    pub sequence: Sequence,
+    pub view: View,
+}
+
+impl Clone for Entry {
+    fn clone(&self) -> Self {
+        Self {
+            key: Arc::clone(&self.key),
+            view: self.view.clone(),
+            key_len: self.key_len,
+            value_len: self.value_len,
+            sequence: self.sequence,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Flusher<K, D, EP, EL>
+where
+    K: Key,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    region_manager: Arc<RegionManager<D, EP, EL>>,
+
+    catalog: Arc<Catalog<K>>,
+
+    buffer: FlushBuffer<D>,
+
+    entry_rx: mpsc::UnboundedReceiver<Entry>,
+
+    _rate_limiter: Option<Arc<RateLimiter>>,
+
+    metrics: Arc<Metrics>,
+
+    stop_rx: broadcast::Receiver<()>,
+}
+
+impl<K, D, EP, EL> Flusher<K, D, EP, EL>
+where
+    K: Key,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    pub fn new(
+        region_manager: Arc<RegionManager<D, EP, EL>>,
+        catalog: Arc<Catalog<K>>,
+        device: D,
+        entry_rx: mpsc::UnboundedReceiver<Entry>,
+        rate_limiter: Option<Arc<RateLimiter>>,
+        metrics: Arc<Metrics>,
+        stop_rx: broadcast::Receiver<()>,
+    ) -> Self {
+        let buffer = FlushBuffer::new(device.clone());
+        Self {
+            region_manager,
+            catalog,
+            buffer,
+            entry_rx,
+            _rate_limiter: rate_limiter,
+            metrics,
+            stop_rx,
+        }
+    }
+
+    pub async fn run(mut self) -> Result<()> {
+        loop {
+            tokio::select! {
+                biased;
+                entry = self.entry_rx.recv() => {
+                    let Some(entry) = entry else {
+                        self.buffer.flush().await?;
+                        tracing::info!("[flusher] exit");
+                        return Ok(());
+                    };
+                    self.handle(entry).await?;
+                }
+                _ = self.stop_rx.recv() => {
+                    self.buffer.flush().await?;
+                    tracing::info!("[flusher] exit");
+                    return Ok(())
+                }
+            }
+        }
+    }
+
+    async fn handle(&mut self, entry: Entry) -> Result<()> {
+        let old_region = self.buffer.region();
+
+        let entry = match self.buffer.write(entry).await {
+            Err(BufferError::NotEnough { entry }) => entry,
+
+            Ok(entries) => return self.update_catalog(entries).await,
+            Err(e) => return Err(Error::from(e)),
+        };
+
+        // current region is full, rotate flush buffer region and retry
+
+        // 1. get a clean region
+        let timer = self
+            .metrics
+            .inner_op_duration_acquire_clean_region
+            .start_timer();
+        let new_region = self
+            .region_manager
+            .clean_regions()
+            .acquire()
+            .instrument(tracing::debug_span!("acquire_clean_region"))
+            .await;
+        drop(timer);
+
+        // 2. rotate flush buffer
+        let entries = self.buffer.rotate(new_region).await?;
+        self.update_catalog(entries).await?;
+        if let Some(old_region) = old_region {
+            self.region_manager.eviction_push(old_region);
+        }
+
+        self.metrics.total_bytes.add(
+            self.region_manager
+                .region(&new_region)
+                .device()
+                .region_size() as u64,
+        );
+
+        // 3. retry write
+        let entries = self.buffer.write(entry).await?;
+        self.update_catalog(entries).await?;
+
+        Ok(())
+    }
+
+    async fn update_catalog(&self, entries: Vec<PositionedEntry>) -> Result<()> {
+        for PositionedEntry {
+            entry:
+                Entry {
+                    key,
+                    view,
+                    key_len,
+                    value_len,
+                    sequence,
+                },
+            region,
+            offset,
+        } in entries
+        {
+            let key = key.downcast::<K>().unwrap();
+            let index = Index::Region {
+                region,
+                version: 0,
+                offset: offset as u32,
+                len: view.aligned() as u32,
+                key_len: key_len as u32,
+                value_len: value_len as u32,
+            };
+            let item = Item { sequence, index };
+            self.catalog.insert(key, item);
+        }
+        Ok(())
+    }
+}
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 402a1195..6fec83be 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -30,23 +30,24 @@ use futures::future::try_join_all;
 use itertools::Itertools;
 use parking_lot::Mutex;
 use tokio::{
-    sync::{broadcast, Semaphore},
+    sync::{broadcast, mpsc, Semaphore},
     task::JoinHandle,
 };
 use twox_hash::XxHash64;
 
 use crate::{
     admission::AdmissionPolicy,
-    catalog::{Catalog, Index, IndexInfo, Sequence},
+    catalog::{Catalog, Index, Item, Sequence},
     device::Device,
     error::Result,
-    flusher::Flusher,
+    flusher_v2::{Entry, Flusher},
     judge::Judges,
     metrics::{Metrics, METRICS},
     reclaimer::Reclaimer,
     region::{Region, RegionHeader, RegionId, REGION_MAGIC},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::ReinsertionPolicy,
+    ring::RingBuffer,
     storage::{Storage, StorageWriter},
 };
 use foyer_common::code::{Key, Value};
@@ -79,6 +80,9 @@ where
     ///       (buffer count = buffer pool size / device region size)
     pub allocator_bits: usize,
 
+    /// `ring_buffer_capacity` will be aligned up to device align.
+    pub ring_buffer_capacity: usize,
+
     /// Catalog indices sharding bits.
     pub catalog_bits: usize,
 
@@ -127,6 +131,7 @@ where
             .field("eviction_config", &self.eviction_config)
             .field("device_config", &self.device_config)
             .field("allocator_bits", &self.allocator_bits)
+            .field("ring_buffer_capacity", &self.ring_buffer_capacity)
             .field("catalog_bits", &self.catalog_bits)
             .field("admissions", &self.admissions)
             .field("reinsertions", &self.reinsertions)
@@ -155,6 +160,7 @@ where
             eviction_config: self.eviction_config.clone(),
             device_config: self.device_config.clone(),
             allocator_bits: self.allocator_bits,
+            ring_buffer_capacity: self.ring_buffer_capacity,
             catalog_bits: self.catalog_bits,
             admissions: self.admissions.clone(),
             reinsertions: self.reinsertions.clone(),
@@ -207,15 +213,17 @@ where
     EL: Link,
 {
     sequence: AtomicU64,
-    indices: Arc<Catalog<K>>,
+    catalog: Arc<Catalog<K>>,
 
     region_manager: Arc<RegionManager<D, EP, EL>>,
+    ring: Arc<RingBuffer<D::IoBufferAllocator>>,
 
     device: D,
 
     admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
     reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
 
+    flusher_entry_txs: Vec<mpsc::UnboundedSender<Entry>>,
     flusher_handles: Mutex<Vec<JoinHandle<()>>>,
     flushers_stop_tx: broadcast::Sender<()>,
 
@@ -241,6 +249,7 @@ where
         let metrics = Arc::new(METRICS.foyer(&config.name));
 
         let device = D::open(config.device_config).await?;
+        assert!(device.regions() >= config.flushers * 2);
 
         let buffer_count = config.buffer_pool_size / device.region_size();
 
@@ -251,6 +260,12 @@ where
             .into());
         }
 
+        let ring = Arc::new(RingBuffer::new_in(
+            device.align(),
+            config.ring_buffer_capacity,
+            device.io_buffer_allocator().clone(),
+        ));
+
         let region_manager = Arc::new(RegionManager::new(
             config.allocator_bits,
             buffer_count,
@@ -261,25 +276,33 @@ where
             metrics.clone(),
         ));
 
-        let indices = Arc::new(Catalog::new(device.regions(), config.catalog_bits));
+        let catalog = Arc::new(Catalog::new(device.regions(), config.catalog_bits));
 
         let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
-        let (reclaimers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
-
         let flusher_stop_rxs = (0..config.flushers)
             .map(|_| flushers_stop_tx.subscribe())
             .collect_vec();
+        let (flusher_entry_txs, flusher_entry_rxs): (
+            Vec<mpsc::UnboundedSender<Entry>>,
+            Vec<mpsc::UnboundedReceiver<Entry>>,
+        ) = (0..config.flushers)
+            .map(|_| mpsc::unbounded_channel())
+            .unzip();
+
+        let (reclaimers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
         let reclaimer_stop_rxs = (0..config.reclaimers)
             .map(|_| reclaimers_stop_tx.subscribe())
             .collect_vec();
 
         let inner = GenericStoreInner {
             sequence: AtomicU64::new(0),
-            indices: indices.clone(),
+            catalog: catalog.clone(),
             region_manager: region_manager.clone(),
+            ring,
             device: device.clone(),
             admissions: config.admissions,
             reinsertions: config.reinsertions,
+            flusher_entry_txs,
             flusher_handles: Mutex::new(vec![]),
             reclaimer_handles: Mutex::new(vec![]),
             flushers_stop_tx,
@@ -292,16 +315,17 @@ where
         };
 
         for admission in store.inner.admissions.iter() {
-            admission.init(&store.inner.indices);
+            admission.init(&store.inner.catalog);
         }
         for reinsertion in store.inner.reinsertions.iter() {
-            reinsertion.init(&store.inner.indices);
+            reinsertion.init(&store.inner.catalog);
         }
 
         let flush_rate_limiter = match config.flush_rate_limit {
             0 => None,
             rate => Some(Arc::new(RateLimiter::new(rate as f64))),
         };
+
         let reclaim_rate_limiter = match config.reclaim_rate_limit {
             0 => None,
             rate => Some(Arc::new(RateLimiter::new(rate as f64))),
@@ -309,15 +333,20 @@ where
 
         let flushers = flusher_stop_rxs
             .into_iter()
-            .map(|stop_rx| {
+            .zip_eq(flusher_entry_rxs.into_iter())
+            .map(|(stop_rx, entry_rx)| {
                 Flusher::new(
                     region_manager.clone(),
+                    catalog.clone(),
+                    device.clone(),
+                    entry_rx,
                     flush_rate_limiter.clone(),
                     metrics.clone(),
                     stop_rx,
                 )
             })
             .collect_vec();
+
         let reclaimers = reclaimer_stop_rxs
             .into_iter()
             .map(|stop_rx| {
@@ -339,7 +368,6 @@ where
             .into_iter()
             .map(|flusher| tokio::spawn(async move { flusher.run().await.unwrap() }))
             .collect_vec();
-
         let reclaimer_handles = reclaimers
             .into_iter()
             .map(|reclaimer| tokio::spawn(async move { reclaimer.run().await.unwrap() }))
@@ -355,19 +383,19 @@ where
         // seal current dirty buffer and trigger flushing
         self.seal().await;
 
-        // stop and wait for reclaimers
-        let handles = self.inner.reclaimer_handles.lock().drain(..).collect_vec();
+        // stop and wait for flushers
+        let handles = self.inner.flusher_handles.lock().drain(..).collect_vec();
         if !handles.is_empty() {
-            self.inner.reclaimers_stop_tx.send(()).unwrap();
+            self.inner.flushers_stop_tx.send(()).unwrap();
         }
         for handle in handles {
             handle.await.unwrap();
         }
 
-        // stop and wait for flushers
-        let handles = self.inner.flusher_handles.lock().drain(..).collect_vec();
+        // stop and wait for reclaimers
+        let handles = self.inner.reclaimer_handles.lock().drain(..).collect_vec();
         if !handles.is_empty() {
-            self.inner.flushers_stop_tx.send(()).unwrap();
+            self.inner.reclaimers_stop_tx.send(()).unwrap();
         }
         for handle in handles {
             handle.await.unwrap();
@@ -384,15 +412,15 @@ where
 
     #[tracing::instrument(skip(self))]
     fn exists(&self, key: &K) -> Result<bool> {
-        Ok(self.inner.indices.lookup(key).is_some())
+        Ok(self.inner.catalog.lookup(key).is_some())
     }
 
     #[tracing::instrument(skip(self))]
     async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
 
-        let info = match self.inner.indices.lookup(key) {
-            Some(info) => info,
+        let item = match self.inner.catalog.lookup(key) {
+            Some(item) => item,
             None => {
                 self.inner
                     .metrics
@@ -402,8 +430,25 @@ where
             }
         };
 
-        match info.index {
-            crate::catalog::Index::RingBuffer {} => todo!(),
+        match item.index {
+            crate::catalog::Index::RingBuffer { view } => {
+                let res = match read_entry::<K, V>(view.as_ref()) {
+                    Some((_key, value)) => Ok(Some(value)),
+                    None => {
+                        // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
+                        self.inner.catalog.remove(key);
+                        Ok(None)
+                    }
+                };
+
+                self.inner
+                    .metrics
+                    .op_duration_lookup_hit
+                    .observe(now.elapsed().as_secs_f64());
+
+                res
+            }
+            // read from region
             crate::catalog::Index::Region {
                 region,
                 version,
@@ -422,7 +467,7 @@ where
                     Some(slice) => slice,
                     None => {
                         // Remove index if the storage layer fails to lookup it (because of region version mismatch).
-                        self.inner.indices.remove(key);
+                        self.inner.catalog.remove(key);
                         self.inner
                             .metrics
                             .op_duration_lookup_miss
@@ -430,6 +475,7 @@ where
                         return Ok(None);
                     }
                 };
+
                 self.inner
                     .metrics
                     .op_bytes_lookup
@@ -439,7 +485,7 @@ where
                     Some((_key, value)) => Ok(Some(value)),
                     None => {
                         // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
-                        self.inner.indices.remove(key);
+                        self.inner.catalog.remove(key);
                         Ok(None)
                     }
                 };
@@ -459,14 +505,14 @@ where
     fn remove(&self, key: &K) -> Result<bool> {
         let _timer = self.inner.metrics.op_duration_remove.start_timer();
 
-        let res = self.inner.indices.remove(key).is_some();
+        let res = self.inner.catalog.remove(key).is_some();
 
         Ok(res)
     }
 
     #[tracing::instrument(skip(self))]
     fn clear(&self) -> Result<()> {
-        self.inner.indices.clear();
+        self.inner.catalog.clear();
 
         // TODO(MrCroxx): set all regions as clean?
 
@@ -474,7 +520,7 @@ where
     }
 
     pub(crate) fn catalog(&self) -> &Arc<Catalog<K>> {
-        &self.inner.indices
+        &self.inner.catalog
     }
 
     pub(crate) fn reinsertions(&self) -> &Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>> {
@@ -501,7 +547,7 @@ where
         for region_id in 0..self.inner.device.regions() as RegionId {
             let semaphore = semaphore.clone();
             let region_manager = self.inner.region_manager.clone();
-            let indices = self.inner.indices.clone();
+            let indices = self.inner.catalog.clone();
             let handle = tokio::spawn(async move {
                 let permit = semaphore.acquire().await;
                 let res = Self::recover_region(region_id, region_manager, indices).await;
@@ -542,14 +588,14 @@ where
     async fn recover_region(
         region_id: RegionId,
         region_manager: Arc<RegionManager<D, EP, EL>>,
-        indices: Arc<Catalog<K>>,
+        catalog: Arc<Catalog<K>>,
     ) -> Result<Option<Sequence>> {
         let region = region_manager.region(&region_id).clone();
         let mut sequence = 0;
         let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
-            while let Some((key, info)) = iter.next().await? {
-                sequence = std::cmp::max(sequence, info.sequence);
-                indices.insert(key, info);
+            while let Some((key, item)) = iter.next().await? {
+                sequence = std::cmp::max(sequence, item.sequence);
+                catalog.insert(Arc::new(key), item);
             }
             region_manager.eviction_push(region_id);
             Some(sequence)
@@ -610,33 +656,57 @@ where
             .op_bytes_insert
             .inc_by(serialized_len as u64);
 
-        let mut slice = match self
-            .inner
-            .region_manager
-            .allocate(serialized_len, !writer.is_skippable)
-            .await
-        {
-            Some(slice) => slice,
-            // Only reachable when writer is skippable.
-            None => return Ok(false),
-        };
-
-        write_entry(slice.as_mut(), &key, &value, sequence);
-
-        let info = IndexInfo {
-            sequence,
-            index: Index::Region {
-                region: slice.region_id(),
-                version: slice.version(),
-                offset: slice.offset() as u32,
-                len: slice.len() as u32,
-                key_len: key.serialized_len() as u32,
-                value_len: value.serialized_len() as u32,
+        // let mut slice = match self
+        //     .inner
+        //     .region_manager
+        //     .allocate(serialized_len, !writer.is_skippable)
+        //     .await
+        // {
+        //     Some(slice) => slice,
+        //     // Only reachable when writer is skippable.
+        //     None => return Ok(false),
+        // };
+
+        // write_entry(slice.as_mut(), &key, &value, sequence);
+
+        // let info = IndexInfo {
+        //     sequence,
+        //     index: Index::Region {
+        //         region: slice.region_id(),
+        //         version: slice.version(),
+        //         offset: slice.offset() as u32,
+        //         len: slice.len() as u32,
+        //         key_len: key.serialized_len() as u32,
+        //         value_len: value.serialized_len() as u32,
+        //     },
+        // };
+        // drop(slice);
+
+        let mut view = self.inner.ring.allocate(serialized_len, sequence).await;
+        let written = write_entry(&mut view, &key, &value, sequence);
+        view.shrink_to(written);
+        let view = view.freeze();
+
+        let key = Arc::new(key);
+
+        self.inner.catalog.insert(
+            key.clone(),
+            Item {
+                sequence,
+                index: Index::RingBuffer { view: view.clone() },
             },
-        };
-        drop(slice);
+        );
 
-        self.inner.indices.insert(key, info);
+        let flusher = sequence as usize % self.inner.flusher_entry_txs.len();
+        self.inner.flusher_entry_txs[flusher]
+            .send(Entry {
+                key_len: key.serialized_len(),
+                value_len: value.serialized_len(),
+                sequence,
+                key,
+                view,
+            })
+            .unwrap();
 
         let duration = now.elapsed() + writer.duration;
         self.inner
@@ -838,7 +908,7 @@ impl EntryHeader {
 /// # Safety
 ///
 /// `buf.len()` must excatly fit entry size
-fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V, sequence: Sequence)
+fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V, sequence: Sequence) -> usize
 where
     K: Key,
     V: Value,
@@ -857,6 +927,7 @@ where
         checksum,
     };
     header.write(&mut buf[..EntryHeader::serialized_len()]);
+    offset
 }
 
 /// | header | value | key | <padding> |
@@ -937,7 +1008,7 @@ where
         }))
     }
 
-    pub async fn next(&mut self) -> Result<Option<(K, IndexInfo)>> {
+    pub async fn next(&mut self) -> Result<Option<(K, Item)>> {
         let region_size = self.region.device().region_size();
         let align = self.region.device().align();
 
@@ -995,7 +1066,7 @@ where
             key
         };
 
-        let info = IndexInfo {
+        let info = Item {
             sequence: header.sequence,
             index: Index::Region {
                 region: self.region.id(),
@@ -1159,6 +1230,7 @@ mod tests {
                 io_size: 4 * KB,
             },
             allocator_bits: 1,
+            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions,
             reinsertions,
@@ -1179,7 +1251,8 @@ mod tests {
         // [3, 4, 5]
         // [6, 7, 8]
         // [9, 10, 11]
-        for i in 0..20 {
+        // ... ...
+        for i in 0..21 {
             store.insert(i, vec![i as u8; 1 * MB]).await.unwrap();
         }
 
@@ -1187,7 +1260,7 @@ mod tests {
 
         let remains = recorder.remains();
 
-        for i in 0..20 {
+        for i in 0..21 {
             if remains.contains(&i) {
                 assert_eq!(
                     store.lookup(&i).await.unwrap().unwrap(),
@@ -1211,6 +1284,7 @@ mod tests {
                 io_size: 4096 * KB,
             },
             allocator_bits: 1,
+            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
@@ -1225,7 +1299,7 @@ mod tests {
         };
         let store = TestStore::open(config).await.unwrap();
 
-        for i in 0..12 {
+        for i in 0..21 {
             if remains.contains(&i) {
                 assert_eq!(
                     store.lookup(&i).await.unwrap().unwrap(),
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 555a60e2..9fd4bb6f 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -239,6 +239,7 @@ mod tests {
                 io_size: 4096 * KB,
             },
             allocator_bits: 1,
+            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
@@ -275,6 +276,7 @@ mod tests {
                 io_size: 4096 * KB,
             },
             allocator_bits: 1,
+            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 49b31473..eaafa1be 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -23,10 +23,12 @@
 #![feature(associated_type_defaults)]
 
 pub mod admission;
+pub mod buffer;
 pub mod catalog;
 pub mod device;
 pub mod error;
 pub mod flusher;
+pub mod flusher_v2;
 pub mod generic;
 pub mod judge;
 pub mod lazy;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index d75f6fd5..61f485dc 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -209,7 +209,7 @@ where
             }
         }
 
-        // step 3: set region last block zero
+        // step 3: wipe region header
         let align = region.device().align();
         let mut buf = region.device().io_buffer(align, align);
         (&mut buf[..]).put_slice(&vec![0; align]);
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index 39bf6d6a..fae2a696 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -29,7 +29,7 @@ where
     K: Key,
     V: Value,
 {
-    indices: OnceLock<Arc<Catalog<K>>>,
+    catalog: OnceLock<Arc<Catalog<K>>>,
     _marker: PhantomData<V>,
 }
 
@@ -40,7 +40,7 @@ where
 {
     fn default() -> Self {
         Self {
-            indices: OnceLock::new(),
+            catalog: OnceLock::new(),
             _marker: PhantomData,
         }
     }
@@ -56,7 +56,7 @@ where
     type Value = V;
 
     fn init(&self, indices: &Arc<Catalog<Self::Key>>) {
-        self.indices.get_or_init(|| indices.clone());
+        self.catalog.get_or_init(|| indices.clone());
     }
 
     fn judge(
@@ -65,7 +65,7 @@ where
         _weight: usize,
         _metrics: &std::sync::Arc<crate::metrics::Metrics>,
     ) -> bool {
-        let indices = self.indices.get().unwrap();
+        let indices = self.catalog.get().unwrap();
         indices.lookup(key).is_some()
     }
 
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index 0262609c..db3dd208 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -17,7 +17,8 @@ use foyer_common::{bits::align_up, continuum::ContinuumUsize};
 use itertools::Itertools;
 use std::{
     alloc::Global,
-    ops::{Deref, DerefMut},
+    fmt::Debug,
+    ops::{Deref, DerefMut, Range},
     sync::{
         atomic::{AtomicUsize, Ordering},
         Arc,
@@ -25,7 +26,6 @@ use std::{
     time::Duration,
 };
 
-#[derive(Debug)]
 pub struct RingBuffer<A = Global>
 where
     A: BufferAllocator,
@@ -42,12 +42,24 @@ where
     continuum: Arc<ContinuumUsize>,
 }
 
+impl<A> Debug for RingBuffer<A>
+where
+    A: BufferAllocator,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RingBuffer")
+            .field("align", &self.align)
+            .field("capacity", &self.capacity)
+            .field("blocks", &self.blocks)
+            .field("allocated", &self.allocated)
+            .finish()
+    }
+}
+
 impl RingBuffer<Global> {
     /// `align` must be power of 2.
-    ///
-    /// `capacity` must be a multiplier of `align`.
-    pub fn new(align: usize, blocks: usize) -> Self {
-        Self::new_in(align, blocks, Global)
+    pub fn new(align: usize, capacity: usize) -> Self {
+        Self::new_in(align, capacity, Global)
     }
 }
 
@@ -56,22 +68,16 @@ where
     A: BufferAllocator,
 {
     /// `align` must be power of 2.
-    ///
-    /// `blocks` must be power of 2.
-    ///
-    /// `capacity = align * blocks`.
-    pub fn new_in(align: usize, blocks: usize, alloc: A) -> Self {
+    pub fn new_in(align: usize, capacity: usize, alloc: A) -> Self {
         assert!(align.is_power_of_two());
-        assert!(blocks.is_power_of_two());
-
-        let capacity = align * blocks;
+        let capacity = align_up(align, capacity);
+        let blocks = capacity / align;
 
         let mut data = Vec::with_capacity_in(capacity, alloc);
         unsafe { data.set_len(capacity) };
 
         let allocated = AtomicUsize::new(0);
 
-        let blocks = capacity / align;
         let continuum = Arc::new(ContinuumUsize::new(blocks));
         let refs = (0..blocks)
             .map(|_| Arc::new(AtomicUsize::default()))
@@ -93,7 +99,7 @@ where
     /// Returns `None` when the allocated buffer cross the boundary.
     ///
     /// When all views from an allocation are dropped, the buffer will be released.
-    pub async fn allocate(self: &Arc<Self>, len: usize, sequence: Sequence) -> ViewMut<A> {
+    pub async fn allocate(self: &Arc<Self>, len: usize, sequence: Sequence) -> ViewMut {
         loop {
             if let Some(view) = self.allocate_inner(len, sequence).await {
                 return view;
@@ -101,11 +107,7 @@ where
         }
     }
 
-    async fn allocate_inner(
-        self: &Arc<Self>,
-        len: usize,
-        sequence: Sequence,
-    ) -> Option<ViewMut<A>> {
+    async fn allocate_inner(self: &Arc<Self>, len: usize, sequence: Sequence) -> Option<ViewMut> {
         let len = align_up(self.align, len);
         let offset = self.allocated.fetch_add(len, Ordering::SeqCst);
 
@@ -128,11 +130,50 @@ where
         }
 
         let refs = self.refs(sequence);
-        Some(ViewMut::new(self, offset, len, refs))
+
+        let ring = Arc::clone(self);
+        let ring: Arc<dyn Ring> = ring;
+        Some(ViewMut::new(ring, offset, len, refs))
     }
 
     fn refs(&self, sequence: Sequence) -> &Arc<AtomicUsize> {
-        &self.refs[sequence as usize & (self.blocks - 1)]
+        &self.refs[sequence as usize % self.blocks]
+    }
+
+    pub fn continuum(&self) -> usize {
+        self.continuum.continuum() * self.align
+    }
+
+    pub fn advance(&self) {
+        self.continuum.advance();
+    }
+}
+
+pub trait Ring: Send + Sync + 'static + Debug {
+    fn align(&self) -> usize;
+
+    fn ptr(&self, offset: usize) -> *mut u8;
+
+    fn release(&self, range: Range<usize>);
+}
+
+impl<A> Ring for RingBuffer<A>
+where
+    A: BufferAllocator,
+{
+    fn align(&self) -> usize {
+        self.align
+    }
+
+    fn ptr(&self, offset: usize) -> *mut u8 {
+        (self.data.as_ptr() as usize + (offset % self.capacity)) as *mut u8
+    }
+
+    fn release(&self, range: Range<usize>) {
+        let start = range.start / self.align;
+        let end = range.end / self.align;
+        debug_assert!(self.continuum.is_vacant(start));
+        self.continuum.submit(start..end);
     }
 }
 
@@ -140,41 +181,47 @@ where
 ///
 /// The underlying buffer of [`ViewMut`] must be valid during its lifetime.
 #[derive(Debug)]
-pub struct ViewMut<A = Global>
-where
-    A: BufferAllocator,
-{
-    ring: Arc<RingBuffer<A>>,
+pub struct ViewMut {
+    ring: Arc<dyn Ring>,
     ptr: *mut u8,
     offset: usize,
     len: usize,
+    capacity: usize,
     refs: Arc<AtomicUsize>,
 }
 
-impl<A> ViewMut<A>
-where
-    A: BufferAllocator,
-{
-    fn new(ring: &Arc<RingBuffer<A>>, offset: usize, len: usize, refs: &Arc<AtomicUsize>) -> Self {
+impl ViewMut {
+    fn new(ring: Arc<dyn Ring>, offset: usize, len: usize, refs: &Arc<AtomicUsize>) -> Self {
         refs.fetch_add(1, Ordering::AcqRel);
+        let ptr = ring.ptr(offset);
         Self {
-            ring: Arc::clone(ring),
-            ptr: (ring.data.as_ptr() as usize + (offset & (ring.capacity - 1))) as *mut u8,
+            ring,
+            ptr,
             offset,
             len,
+            capacity: len,
             refs: Arc::clone(refs),
         }
     }
 
-    pub fn freeze(self) -> View<A> {
+    /// Shrink the accessable area of [`ViewMut`].
+    ///
+    /// `shrink` will not release the underlying buffer size.
+    pub fn shrink_to(&mut self, len: usize) {
+        debug_assert!(len <= self.capacity);
+        self.len = len;
+    }
+
+    pub fn aligned(&self) -> usize {
+        align_up(self.ring.align(), self.len)
+    }
+
+    pub fn freeze(self) -> View {
         View::from(self)
     }
 }
 
-impl<A> Deref for ViewMut<A>
-where
-    A: BufferAllocator,
-{
+impl Deref for ViewMut {
     type Target = [u8];
 
     fn deref(&self) -> &Self::Target {
@@ -182,56 +229,38 @@ where
     }
 }
 
-impl<A> DerefMut for ViewMut<A>
-where
-    A: BufferAllocator,
-{
+impl DerefMut for ViewMut {
     fn deref_mut(&mut self) -> &mut Self::Target {
         unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) }
     }
 }
 
-impl<A> Drop for ViewMut<A>
-where
-    A: BufferAllocator,
-{
+impl Drop for ViewMut {
     fn drop(&mut self) {
         if self.refs.fetch_sub(1, Ordering::AcqRel) == 1 {
-            let block_start = self.offset / self.ring.align;
-            let block_end = (self.offset + self.len) / self.ring.align;
-            self.ring.continuum.is_vacant(block_start);
-            self.ring.continuum.submit(block_start..block_end);
+            self.ring.release(self.offset..self.offset + self.capacity);
         }
     }
 }
 
-unsafe impl<A> Send for ViewMut<A> where A: BufferAllocator {}
-unsafe impl<A> Sync for ViewMut<A> where A: BufferAllocator {}
+unsafe impl Send for ViewMut {}
+unsafe impl Sync for ViewMut {}
 
 /// # Safety
 ///
 /// The underlying buffer of [`View`] must be valid during its lifetime.
 #[derive(Debug)]
-pub struct View<A = Global>
-where
-    A: BufferAllocator,
-{
-    view: ViewMut<A>,
+pub struct View {
+    view: ViewMut,
 }
 
-impl<A> From<ViewMut<A>> for View<A>
-where
-    A: BufferAllocator,
-{
-    fn from(view: ViewMut<A>) -> Self {
+impl From<ViewMut> for View {
+    fn from(view: ViewMut) -> Self {
         Self { view }
     }
 }
 
-impl<A> Deref for View<A>
-where
-    A: BufferAllocator,
-{
+impl Deref for View {
     type Target = [u8];
 
     fn deref(&self) -> &Self::Target {
@@ -239,10 +268,7 @@ where
     }
 }
 
-impl<A> Clone for View<A>
-where
-    A: BufferAllocator,
-{
+impl Clone for View {
     fn clone(&self) -> Self {
         self.view.refs.fetch_add(1, Ordering::AcqRel);
         let view = ViewMut {
@@ -250,14 +276,21 @@ where
             ptr: self.view.ptr,
             offset: self.view.offset,
             len: self.view.len,
+            capacity: self.view.capacity,
             refs: self.view.refs.clone(),
         };
         Self { view }
     }
 }
 
-unsafe impl<A> Send for View<A> where A: BufferAllocator {}
-unsafe impl<A> Sync for View<A> where A: BufferAllocator {}
+impl View {
+    pub fn aligned(&self) -> usize {
+        self.view.aligned()
+    }
+}
+
+unsafe impl Send for View {}
+unsafe impl Sync for View {}
 
 #[cfg(test)]
 mod tests {
@@ -281,9 +314,9 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     async fn test_ring() {
         const ALIGN: usize = 4096; // 4 KiB
-        const BLOCKS: usize = 4096; // capacity = 4 KiB * 4K = 16 MiB
+        const CAPACITY: usize = 16 * 1024 * 1024; // 16 MiB
 
-        let ring = Arc::new(RingBuffer::new(ALIGN, BLOCKS));
+        let ring = Arc::new(RingBuffer::new(ALIGN, CAPACITY));
         let sequence = Arc::new(AtomicU64::default());
 
         let mut views = BTreeMap::new();
@@ -294,7 +327,7 @@ mod tests {
                 .await
                 .unwrap();
             assert_eq!(view.offset, i * 1024 * 1024);
-            assert_eq!(view.len, 1024 * 1024);
+            assert_eq!(view.capacity, 1024 * 1024);
             views.insert(seq, view);
         }
         let seq = sequence.fetch_add(1, Ordering::Relaxed);
@@ -311,17 +344,17 @@ mod tests {
         views.remove(&1).unwrap();
         let view = future.await.unwrap();
         assert_eq!(view.offset, 17 * 1024 * 1024);
-        assert_eq!(view.len, 2 * 1024 * 1024);
+        assert_eq!(view.capacity, 2 * 1024 * 1024);
         views.insert(seq, view);
 
         drop(views);
     }
 
-    async fn test_ring_concurrent_case(blocks: usize, concurrency: usize, loops: usize) {
+    async fn test_ring_concurrent_case(capacity: usize, concurrency: usize, loops: usize) {
         const ALIGN: usize = 4096; // 4 KiB
         const SIZE: Range<usize> = 16 * 1024..256 * 1024; // 16 KiB ~ 128 KiB
 
-        let ring = Arc::new(RingBuffer::new(ALIGN, blocks));
+        let ring = Arc::new(RingBuffer::new(ALIGN, capacity));
         let sequence = Arc::new(AtomicU64::default());
 
         let tasks = (0..concurrency)
@@ -351,8 +384,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     async fn test_ring_concurrent_small() {
-        // 4096 * 4096 = 16 MiB
-        test_ring_concurrent_case(4096, 16, 100).await;
+        test_ring_concurrent_case(16 * 1024 * 1024, 16, 100).await;
     }
 
     #[ignore]
@@ -361,7 +393,12 @@ mod tests {
         let concurrency = available_parallelism().unwrap().get() * 64;
 
         let now = Instant::now();
-        test_ring_concurrent_case(65536, available_parallelism().unwrap().get() * 64, 1000).await;
+        test_ring_concurrent_case(
+            128 * 1024 * 1024,
+            available_parallelism().unwrap().get() * 64,
+            1000,
+        )
+        .await;
         let elapsed = now.elapsed();
 
         println!("========== ring current fuzzy begin ==========");
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 51ac9668..58977bb7 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -129,6 +129,7 @@ async fn test_store() {
             io_size: 4 * KB,
         },
         allocator_bits: 0,
+        ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
@@ -160,6 +161,7 @@ async fn test_lazy_store() {
             io_size: 4 * KB,
         },
         allocator_bits: 0,
+        ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
@@ -192,6 +194,7 @@ async fn test_runtime_store() {
                 io_size: 4 * KB,
             },
             allocator_bits: 0,
+            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
@@ -230,6 +233,7 @@ async fn test_runtime_lazy_store() {
                 io_size: 4 * KB,
             },
             allocator_bits: 0,
+            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],

From c36871ec23c6280c2af8b79314f8c8b67cefc139 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 27 Oct 2023 12:31:37 +0800
Subject: [PATCH 155/261] chore: refine metrics for new write model (#189)

* chore: refine metrics for new write model

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add entry flush metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                        | 13 +++++-
 foyer-storage/src/catalog.rs    | 43 +++++++++++++++++---
 foyer-storage/src/flusher_v2.rs | 30 ++++++++++----
 foyer-storage/src/generic.rs    | 70 +++++++++++----------------------
 foyer-storage/src/metrics.rs    | 18 +++++++++
 foyer-storage/src/ring.rs       | 29 +++++++++++++-
 6 files changed, 140 insertions(+), 63 deletions(-)

diff --git a/Makefile b/Makefile
index 117bea23..209fc389 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,17 @@ deps:
 	./scripts/install-deps.sh
 
 check:
+	shellcheck ./scripts/*
+	./.github/template/generate.sh
+	./scripts/minimize-dashboards.sh
+	cargo hakari generate
+	cargo hakari manage-deps
+	cargo sort -w
+	cargo fmt --all
+	cargo clippy --all-targets
+	cargo udeps --workspace --exclude foyer-workspace-hack
+
+check-all:
 	shellcheck ./scripts/*
 	./.github/template/generate.sh
 	./scripts/minimize-dashboards.sh
@@ -32,7 +43,7 @@ madsim:
 	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" RUST_BACKTRACE=1 cargo nextest run --all
 	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" RUST_BACKTRACE=1 cargo test --doc
 
-all: check test-all
+all: check-all test-all
 
 fast: check test
 
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 17b5a3f5..4eb4c256 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -16,6 +16,7 @@ use std::{
     collections::btree_map::{BTreeMap, Entry},
     hash::Hasher,
     sync::Arc,
+    time::Instant,
 };
 
 use foyer_common::code::Key;
@@ -25,6 +26,7 @@ use twox_hash::XxHash64;
 
 use crate::{
     device::BufferAllocator,
+    metrics::Metrics,
     region::{RegionId, Version},
     ring::View,
 };
@@ -48,8 +50,28 @@ pub enum Index {
 
 #[derive(Debug, Clone)]
 pub struct Item {
-    pub sequence: Sequence,
-    pub index: Index,
+    sequence: Sequence,
+    index: Index,
+
+    inserted: Option<Instant>,
+}
+
+impl Item {
+    pub fn new(sequence: Sequence, index: Index) -> Self {
+        Self {
+            sequence,
+            index,
+            inserted: None,
+        }
+    }
+
+    pub fn sequence(&self) -> &Sequence {
+        &self.sequence
+    }
+
+    pub fn index(&self) -> &Index {
+        &self.index
+    }
 }
 
 #[derive(Debug)]
@@ -65,13 +87,15 @@ where
 
     /// Sharded by region id.
     regions: Vec<Mutex<BTreeMap<Arc<K>, u64>>>,
+
+    metrics: Arc<Metrics>,
 }
 
 impl<K> Catalog<K>
 where
     K: Key,
 {
-    pub fn new(regions: usize, bits: usize) -> Self {
+    pub fn new(regions: usize, bits: usize, metrics: Arc<Metrics>) -> Self {
         let infos = (0..1 << bits)
             .map(|_| RwLock::new(BTreeMap::new()))
             .collect_vec();
@@ -82,10 +106,12 @@ where
             bits,
             items: infos,
             regions,
+
+            metrics,
         }
     }
 
-    pub fn insert(&self, key: Arc<K>, item: Item) {
+    pub fn insert(&self, key: Arc<K>, mut item: Item) {
         // TODO(MrCroxx): compare sequence.
 
         if let Index::Region { region, .. } = item.index {
@@ -96,7 +122,14 @@ where
 
         let shard = self.shard(&key);
         // TODO(MrCroxx): handle old key?
-        let _ = self.items[shard].write().insert(key.clone(), item);
+        let old = {
+            let mut guard = self.items[shard].write();
+            item.inserted = Some(Instant::now());
+            guard.insert(key.clone(), item)
+        };
+        if let Some(old) = old && let Index::RingBuffer { .. } = old.index() {
+            self.metrics.inner_op_duration_entry_flush.observe(old.inserted.unwrap().elapsed().as_secs_f64());
+        }
     }
 
     pub fn lookup(&self, key: &K) -> Option<Item> {
diff --git a/foyer-storage/src/flusher_v2.rs b/foyer-storage/src/flusher_v2.rs
index 14ea0888..9972d0bc 100644
--- a/foyer-storage/src/flusher_v2.rs
+++ b/foyer-storage/src/flusher_v2.rs
@@ -12,12 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use foyer_common::{code::Key, rate::RateLimiter};
-use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use std::{any::Any, sync::Arc};
-use tokio::sync::{broadcast, mpsc};
-use tracing::Instrument;
-
 use crate::{
     buffer::{BufferError, FlushBuffer, PositionedEntry},
     catalog::{Catalog, Index, Item, Sequence},
@@ -27,8 +21,12 @@ use crate::{
     region_manager::{RegionEpItemAdapter, RegionManager},
     ring::View,
 };
+use foyer_common::{code::Key, rate::RateLimiter};
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use std::{any::Any, fmt::Debug, sync::Arc};
+use tokio::sync::{broadcast, mpsc};
+use tracing::Instrument;
 
-#[derive(Debug)]
 pub struct Entry {
     /// # Safety
     ///
@@ -39,9 +37,22 @@ pub struct Entry {
     pub key_len: usize,
     pub value_len: usize,
     pub sequence: Sequence,
+
+    /// Hold a view of referenced buffer, for lookup and prevent from releasing.
     pub view: View,
 }
 
+impl Debug for Entry {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Entry")
+            .field("key_len", &self.key_len)
+            .field("value_len", &self.value_len)
+            .field("sequence", &self.sequence)
+            .field("view", &self.view)
+            .finish()
+    }
+}
+
 impl Clone for Entry {
     fn clone(&self) -> Self {
         Self {
@@ -172,7 +183,9 @@ where
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn update_catalog(&self, entries: Vec<PositionedEntry>) -> Result<()> {
+        let timer = self.metrics.inner_op_duration_update_catalog.start_timer();
         for PositionedEntry {
             entry:
                 Entry {
@@ -195,9 +208,10 @@ where
                 key_len: key_len as u32,
                 value_len: value_len as u32,
             };
-            let item = Item { sequence, index };
+            let item = Item::new(sequence, index);
             self.catalog.insert(key, item);
         }
+        drop(timer);
         Ok(())
     }
 }
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 6fec83be..79b4c8f1 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -260,9 +260,10 @@ where
             .into());
         }
 
-        let ring = Arc::new(RingBuffer::new_in(
+        let ring = Arc::new(RingBuffer::with_metrics_in(
             device.align(),
             config.ring_buffer_capacity,
+            metrics.clone(),
             device.io_buffer_allocator().clone(),
         ));
 
@@ -276,7 +277,11 @@ where
             metrics.clone(),
         ));
 
-        let catalog = Arc::new(Catalog::new(device.regions(), config.catalog_bits));
+        let catalog = Arc::new(Catalog::new(
+            device.regions(),
+            config.catalog_bits,
+            metrics.clone(),
+        ));
 
         let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
         let flusher_stop_rxs = (0..config.flushers)
@@ -430,7 +435,7 @@ where
             }
         };
 
-        match item.index {
+        match item.index() {
             crate::catalog::Index::RingBuffer { view } => {
                 let res = match read_entry::<K, V>(view.as_ref()) {
                     Some((_key, value)) => Ok(Some(value)),
@@ -457,13 +462,13 @@ where
                 key_len: _,
                 value_len: _,
             } => {
-                self.inner.region_manager.record_access(&region);
-                let region = self.inner.region_manager.region(&region);
-                let start = offset as usize;
-                let end = start + len as usize;
+                self.inner.region_manager.record_access(region);
+                let region = self.inner.region_manager.region(region);
+                let start = *offset as usize;
+                let end = start + *len as usize;
 
                 // TODO(MrCroxx): read value only
-                let slice = match region.load(start..end, version).await? {
+                let slice = match region.load(start..end, *version).await? {
                     Some(slice) => slice,
                     None => {
                         // Remove index if the storage layer fails to lookup it (because of region version mismatch).
@@ -594,7 +599,7 @@ where
         let mut sequence = 0;
         let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
             while let Some((key, item)) = iter.next().await? {
-                sequence = std::cmp::max(sequence, item.sequence);
+                sequence = std::cmp::max(sequence, *item.sequence());
                 catalog.insert(Arc::new(key), item);
             }
             region_manager.eviction_push(region_id);
@@ -656,32 +661,6 @@ where
             .op_bytes_insert
             .inc_by(serialized_len as u64);
 
-        // let mut slice = match self
-        //     .inner
-        //     .region_manager
-        //     .allocate(serialized_len, !writer.is_skippable)
-        //     .await
-        // {
-        //     Some(slice) => slice,
-        //     // Only reachable when writer is skippable.
-        //     None => return Ok(false),
-        // };
-
-        // write_entry(slice.as_mut(), &key, &value, sequence);
-
-        // let info = IndexInfo {
-        //     sequence,
-        //     index: Index::Region {
-        //         region: slice.region_id(),
-        //         version: slice.version(),
-        //         offset: slice.offset() as u32,
-        //         len: slice.len() as u32,
-        //         key_len: key.serialized_len() as u32,
-        //         value_len: value.serialized_len() as u32,
-        //     },
-        // };
-        // drop(slice);
-
         let mut view = self.inner.ring.allocate(serialized_len, sequence).await;
         let written = write_entry(&mut view, &key, &value, sequence);
         view.shrink_to(written);
@@ -691,10 +670,7 @@ where
 
         self.inner.catalog.insert(
             key.clone(),
-            Item {
-                sequence,
-                index: Index::RingBuffer { view: view.clone() },
-            },
+            Item::new(sequence, Index::RingBuffer { view: view.clone() }),
         );
 
         let flusher = sequence as usize % self.inner.flusher_entry_txs.len();
@@ -1066,9 +1042,9 @@ where
             key
         };
 
-        let info = Item {
-            sequence: header.sequence,
-            index: Index::Region {
+        let info = Item::new(
+            header.sequence,
+            Index::Region {
                 region: self.region.id(),
                 version: 0,
                 offset: self.cursor as u32,
@@ -1076,7 +1052,7 @@ where
                 key_len: header.key_len,
                 value_len: header.value_len,
             },
-        };
+        );
 
         self.cursor += entry_len;
 
@@ -1084,19 +1060,19 @@ where
     }
 
     pub async fn next_kv(&mut self) -> Result<Option<(K, V)>> {
-        let (_, info) = match self.next().await {
+        let (_, item) = match self.next().await {
             Ok(Some(res)) => res,
             Ok(None) => return Ok(None),
             Err(e) => return Err(e),
         };
 
-        let Index::Region { offset, len, .. } = info.index else {
+        let Index::Region { offset, len, .. } = item.index() else {
             unreachable!("kv loaded from region must have index of region")
         };
 
         // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
-        let start = offset as usize;
-        let end = start + len as usize;
+        let start = *offset as usize;
+        let end = start + *len as usize;
         let Some(slice) = self.region.load(start..end, 0).await? else {
             return Ok(None);
         };
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 01decf3c..241190d1 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -156,6 +156,9 @@ pub struct Metrics {
 
     pub inner_op_duration_acquire_clean_region: Histogram,
     pub inner_op_duration_acquire_clean_buffer: Histogram,
+    pub inner_op_duration_wait_ring_buffer: Histogram,
+    pub inner_op_duration_update_catalog: Histogram,
+    pub inner_op_duration_entry_flush: Histogram,
 }
 
 impl Metrics {
@@ -199,6 +202,18 @@ impl Metrics {
             global
                 .inner_op_duration
                 .with_label_values(&[foyer, "acquire_clean_buffer", ""]);
+        let inner_op_duration_wait_ring_buffer =
+            global
+                .inner_op_duration
+                .with_label_values(&[foyer, "wait_ring_buffer", ""]);
+        let inner_op_duration_update_catalog =
+            global
+                .inner_op_duration
+                .with_label_values(&[foyer, "update_catalog", ""]);
+        let inner_op_duration_entry_flush =
+            global
+                .inner_op_duration
+                .with_label_values(&[foyer, "entry_flush", ""]);
 
         Self {
             op_duration_insert_inserted,
@@ -220,6 +235,9 @@ impl Metrics {
 
             inner_op_duration_acquire_clean_region,
             inner_op_duration_acquire_clean_buffer,
+            inner_op_duration_wait_ring_buffer,
+            inner_op_duration_update_catalog,
+            inner_op_duration_entry_flush,
         }
     }
 }
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index db3dd208..f88187c6 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -12,7 +12,11 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{catalog::Sequence, device::BufferAllocator};
+use crate::{
+    catalog::Sequence,
+    device::BufferAllocator,
+    metrics::{Metrics, METRICS},
+};
 use foyer_common::{bits::align_up, continuum::ContinuumUsize};
 use itertools::Itertools;
 use std::{
@@ -23,7 +27,7 @@ use std::{
         atomic::{AtomicUsize, Ordering},
         Arc,
     },
-    time::Duration,
+    time::{Duration, Instant},
 };
 
 pub struct RingBuffer<A = Global>
@@ -40,6 +44,8 @@ where
     refs: Vec<Arc<AtomicUsize>>,
 
     continuum: Arc<ContinuumUsize>,
+
+    metrics: Arc<Metrics>,
 }
 
 impl<A> Debug for RingBuffer<A>
@@ -61,6 +67,11 @@ impl RingBuffer<Global> {
     pub fn new(align: usize, capacity: usize) -> Self {
         Self::new_in(align, capacity, Global)
     }
+
+    /// `align` must be power of 2.
+    pub fn with_metrics(align: usize, capacity: usize, metrics: Arc<Metrics>) -> Self {
+        Self::with_metrics_in(align, capacity, metrics, Global)
+    }
 }
 
 impl<A> RingBuffer<A>
@@ -69,6 +80,12 @@ where
 {
     /// `align` must be power of 2.
     pub fn new_in(align: usize, capacity: usize, alloc: A) -> Self {
+        let metrics = Arc::new(METRICS.foyer(""));
+        Self::with_metrics_in(align, capacity, metrics, alloc)
+    }
+
+    /// `align` must be power of 2.
+    pub fn with_metrics_in(align: usize, capacity: usize, metrics: Arc<Metrics>, alloc: A) -> Self {
         assert!(align.is_power_of_two());
         let capacity = align_up(align, capacity);
         let blocks = capacity / align;
@@ -91,6 +108,7 @@ where
             allocated,
             refs,
             continuum,
+            metrics,
         }
     }
 
@@ -99,6 +117,7 @@ where
     /// Returns `None` when the allocated buffer cross the boundary.
     ///
     /// When all views from an allocation are dropped, the buffer will be released.
+    #[tracing::instrument(skip(self))]
     pub async fn allocate(self: &Arc<Self>, len: usize, sequence: Sequence) -> ViewMut {
         loop {
             if let Some(view) = self.allocate_inner(len, sequence).await {
@@ -107,12 +126,14 @@ where
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn allocate_inner(self: &Arc<Self>, len: usize, sequence: Sequence) -> Option<ViewMut> {
         let len = align_up(self.align, len);
         let offset = self.allocated.fetch_add(len, Ordering::SeqCst);
 
         debug_assert_eq!(offset & (self.align - 1), 0);
 
+        let now = Instant::now();
         loop {
             self.continuum.advance();
             if self.continuum.continuum() * self.align + self.capacity >= offset + len {
@@ -120,6 +141,10 @@ where
             }
             tokio::time::sleep(Duration::from_micros(100)).await;
         }
+        let elapsed = now.elapsed();
+        self.metrics
+            .inner_op_duration_wait_ring_buffer
+            .observe(elapsed.as_secs_f64());
 
         if offset / self.capacity != (offset + len) / self.capacity {
             debug_assert!(self.continuum.is_vacant(offset / self.align));

From 97c38727c85a438ba30dbf5bf42c1affe19d0f6f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 27 Oct 2023 18:53:34 +0800
Subject: [PATCH 156/261] feat: advance continuum when submit (#190)

* feat: advance continuum when submit

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add flusher handle metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/flusher_v2.rs | 13 ++++++-------
 foyer-storage/src/generic.rs    |  6 ------
 foyer-storage/src/metrics.rs    |  6 ++++++
 foyer-storage/src/ring.rs       |  4 ++--
 4 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/foyer-storage/src/flusher_v2.rs b/foyer-storage/src/flusher_v2.rs
index 9972d0bc..0afe6940 100644
--- a/foyer-storage/src/flusher_v2.rs
+++ b/foyer-storage/src/flusher_v2.rs
@@ -21,7 +21,7 @@ use crate::{
     region_manager::{RegionEpItemAdapter, RegionManager},
     ring::View,
 };
-use foyer_common::{code::Key, rate::RateLimiter};
+use foyer_common::code::Key;
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use std::{any::Any, fmt::Debug, sync::Arc};
 use tokio::sync::{broadcast, mpsc};
@@ -81,8 +81,6 @@ where
 
     entry_rx: mpsc::UnboundedReceiver<Entry>,
 
-    _rate_limiter: Option<Arc<RateLimiter>>,
-
     metrics: Arc<Metrics>,
 
     stop_rx: broadcast::Receiver<()>,
@@ -100,7 +98,6 @@ where
         catalog: Arc<Catalog<K>>,
         device: D,
         entry_rx: mpsc::UnboundedReceiver<Entry>,
-        rate_limiter: Option<Arc<RateLimiter>>,
         metrics: Arc<Metrics>,
         stop_rx: broadcast::Receiver<()>,
     ) -> Self {
@@ -110,7 +107,6 @@ where
             catalog,
             buffer,
             entry_rx,
-            _rate_limiter: rate_limiter,
             metrics,
             stop_rx,
         }
@@ -138,6 +134,8 @@ where
     }
 
     async fn handle(&mut self, entry: Entry) -> Result<()> {
+        let timer = self.metrics.inner_op_duration_flusher_handle.start_timer();
+
         let old_region = self.buffer.region();
 
         let entry = match self.buffer.write(entry).await {
@@ -150,7 +148,7 @@ where
         // current region is full, rotate flush buffer region and retry
 
         // 1. get a clean region
-        let timer = self
+        let acquire_clean_region_timer = self
             .metrics
             .inner_op_duration_acquire_clean_region
             .start_timer();
@@ -160,7 +158,7 @@ where
             .acquire()
             .instrument(tracing::debug_span!("acquire_clean_region"))
             .await;
-        drop(timer);
+        drop(acquire_clean_region_timer);
 
         // 2. rotate flush buffer
         let entries = self.buffer.rotate(new_region).await?;
@@ -180,6 +178,7 @@ where
         let entries = self.buffer.write(entry).await?;
         self.update_catalog(entries).await?;
 
+        drop(timer);
         Ok(())
     }
 
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 79b4c8f1..4d5250e2 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -326,11 +326,6 @@ where
             reinsertion.init(&store.inner.catalog);
         }
 
-        let flush_rate_limiter = match config.flush_rate_limit {
-            0 => None,
-            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
-        };
-
         let reclaim_rate_limiter = match config.reclaim_rate_limit {
             0 => None,
             rate => Some(Arc::new(RateLimiter::new(rate as f64))),
@@ -345,7 +340,6 @@ where
                     catalog.clone(),
                     device.clone(),
                     entry_rx,
-                    flush_rate_limiter.clone(),
                     metrics.clone(),
                     stop_rx,
                 )
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 241190d1..46febfab 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -159,6 +159,7 @@ pub struct Metrics {
     pub inner_op_duration_wait_ring_buffer: Histogram,
     pub inner_op_duration_update_catalog: Histogram,
     pub inner_op_duration_entry_flush: Histogram,
+    pub inner_op_duration_flusher_handle: Histogram,
 }
 
 impl Metrics {
@@ -214,6 +215,10 @@ impl Metrics {
             global
                 .inner_op_duration
                 .with_label_values(&[foyer, "entry_flush", ""]);
+        let inner_op_duration_flusher_handle =
+            global
+                .inner_op_duration
+                .with_label_values(&[foyer, "flusher_handle", ""]);
 
         Self {
             op_duration_insert_inserted,
@@ -238,6 +243,7 @@ impl Metrics {
             inner_op_duration_wait_ring_buffer,
             inner_op_duration_update_catalog,
             inner_op_duration_entry_flush,
+            inner_op_duration_flusher_handle,
         }
     }
 }
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index f88187c6..2306f14d 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -150,7 +150,7 @@ where
             debug_assert!(self.continuum.is_vacant(offset / self.align));
 
             self.continuum
-                .submit((offset / self.align)..((offset + len) / self.align));
+                .submit_advance((offset / self.align)..((offset + len) / self.align));
             return None;
         }
 
@@ -198,7 +198,7 @@ where
         let start = range.start / self.align;
         let end = range.end / self.align;
         debug_assert!(self.continuum.is_vacant(start));
-        self.continuum.submit(start..end);
+        self.continuum.submit_advance(start..end);
     }
 }
 

From 73faf7a2aade5ac6fcb1111771a1bc8403d82ee5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 27 Oct 2023 19:01:52 +0800
Subject: [PATCH 157/261] fix: allow not power of 2 continuum capacity (#191)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 etc/grafana/dashboards/foyer.json |  2 +-
 foyer-common/src/continuum.rs     |  4 +---
 foyer-storage/src/generic.rs      |  4 ++++
 foyer-storage/src/metrics.rs      | 25 ++++++++++++++++++++++---
 foyer-storage/src/ring.rs         | 10 ++++++++++
 5 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/etc/grafana/dashboards/foyer.json b/etc/grafana/dashboards/foyer.json
index de5308ef..2923d0b4 100644
--- a/etc/grafana/dashboards/foyer.json
+++ b/etc/grafana/dashboards/foyer.json
@@ -1 +1 @@
-{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Write Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":10,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Read Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":9,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":25},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":33},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""}
+{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Write Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":10,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Read Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":9,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":25},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":33},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":33},"id":11,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_inner_bytes) by (foyer, component, extra) ","instant":false,"legendFormat":"{{foyer}} {{component}} {{extra}}","range":true,"refId":"A"}],"title":"Inner Size","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""}
diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
index 15e47a8e..fe0bddd9 100644
--- a/foyer-common/src/continuum.rs
+++ b/foyer-common/src/continuum.rs
@@ -29,9 +29,7 @@ macro_rules! def_continuum {
             }
 
             impl $name {
-                /// `capacity` must be power of 2.
                 pub fn new(capacity: $uint) -> Self {
-                    assert!(capacity.is_power_of_two());
                     let slots = (0..capacity).map(|_| <$atomic>::default()).collect_vec();
                     let continuum = <$atomic>::default();
                     Self {
@@ -97,7 +95,7 @@ macro_rules! def_continuum {
                 }
 
                 fn slot(&self, position: $uint) -> usize {
-                    (position & (self.capacity - 1)) as usize
+                    (position % self.capacity) as usize
                 }
 
                 /// `stop: Fn(continuum, next) -> bool`.
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 4d5250e2..a5e1d33b 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -655,6 +655,10 @@ where
             .op_bytes_insert
             .inc_by(serialized_len as u64);
 
+        self.inner
+            .metrics
+            .inner_bytes_ring_buffer_remains
+            .set(self.inner.ring.remains() as i64);
         let mut view = self.inner.ring.allocate(serialized_len, sequence).await;
         let written = write_entry(&mut view, &key, &value, sequence);
         view.shrink_to(written);
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 46febfab..fdf34a44 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -16,8 +16,9 @@ use std::sync::{LazyLock, OnceLock};
 
 use prometheus::{
     core::{AtomicU64, GenericGauge, GenericGaugeVec},
-    opts, register_histogram_vec_with_registry, register_int_counter_vec_with_registry, Histogram,
-    HistogramVec, IntCounter, IntCounterVec, Registry,
+    opts, register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
+    register_int_gauge_vec_with_registry, Histogram, HistogramVec, IntCounter, IntCounterVec,
+    IntGauge, IntGaugeVec, Registry,
 };
 type UintGaugeVec = GenericGaugeVec<AtomicU64>;
 type UintGauge = GenericGauge<AtomicU64>;
@@ -67,6 +68,7 @@ pub struct GlobalMetrics {
     total_bytes: UintGaugeVec,
 
     inner_op_duration: HistogramVec,
+    inner_bytes: IntGaugeVec,
 }
 
 impl Default for GlobalMetrics {
@@ -115,7 +117,15 @@ impl GlobalMetrics {
             "foyer_storage_inner_op_duration",
             "foyer storage inner op duration",
             &["foyer", "op", "extra"],
-            vec![0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0],
+            vec![0.0001, 0.01, 0.02, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 10.0],
+            registry,
+        )
+        .unwrap();
+
+        let inner_bytes = register_int_gauge_vec_with_registry!(
+            "foyer_storage_inner_bytes",
+            "foyer storage inner bytes",
+            &["foyer", "component", "extra"],
             registry,
         )
         .unwrap();
@@ -127,6 +137,7 @@ impl GlobalMetrics {
             total_bytes,
 
             inner_op_duration,
+            inner_bytes,
         }
     }
 
@@ -160,6 +171,8 @@ pub struct Metrics {
     pub inner_op_duration_update_catalog: Histogram,
     pub inner_op_duration_entry_flush: Histogram,
     pub inner_op_duration_flusher_handle: Histogram,
+
+    pub inner_bytes_ring_buffer_remains: IntGauge,
 }
 
 impl Metrics {
@@ -220,6 +233,10 @@ impl Metrics {
                 .inner_op_duration
                 .with_label_values(&[foyer, "flusher_handle", ""]);
 
+        let inner_bytes_ring_buffer_remains = global
+            .inner_bytes
+            .with_label_values(&[foyer, "ring", "remains"]);
+
         Self {
             op_duration_insert_inserted,
             op_duration_insert_filtered,
@@ -244,6 +261,8 @@ impl Metrics {
             inner_op_duration_update_catalog,
             inner_op_duration_entry_flush,
             inner_op_duration_flusher_handle,
+
+            inner_bytes_ring_buffer_remains,
         }
     }
 }
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index 2306f14d..2b1025ad 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -169,9 +169,19 @@ where
         self.continuum.continuum() * self.align
     }
 
+    pub fn allocated(&self) -> usize {
+        self.allocated.load(Ordering::Relaxed)
+    }
+
     pub fn advance(&self) {
         self.continuum.advance();
     }
+
+    /// Reutrns positive number means how many bytes remains,
+    /// returns negative number means how many bytes are waiting for allocation.
+    pub fn remains(&self) -> isize {
+        self.continuum() as isize - self.allocated() as isize
+    }
 }
 
 pub trait Ring: Send + Sync + 'static + Debug {

From 524a2ee65c3e2fc7f3fa3b2a82e1451aac88a643 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 30 Oct 2023 23:16:41 +0800
Subject: [PATCH 158/261] refactor: cleanup unnecessary (#193)

* chore: remove uncessary code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove more uncessary code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: more cleanup

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/buffer.rs         |   2 +-
 foyer-storage/src/catalog.rs        |   8 +-
 foyer-storage/src/flusher.rs        | 212 +++++++++++------
 foyer-storage/src/flusher_v2.rs     | 216 ------------------
 foyer-storage/src/generic.rs        |  28 +--
 foyer-storage/src/lazy.rs           |   1 +
 foyer-storage/src/lib.rs            |   1 -
 foyer-storage/src/reclaimer.rs      |   8 +-
 foyer-storage/src/region.rs         | 340 +++-------------------------
 foyer-storage/src/region_manager.rs | 134 +----------
 10 files changed, 186 insertions(+), 764 deletions(-)
 delete mode 100644 foyer-storage/src/flusher_v2.rs

diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index bd6132e6..5027cb60 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -16,7 +16,7 @@ use foyer_common::bits::{align_up, is_aligned};
 
 use crate::{
     device::{error::DeviceError, Device},
-    flusher_v2::Entry,
+    flusher::Entry,
     region::{RegionHeader, RegionId, REGION_MAGIC},
 };
 
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 4eb4c256..92dd5c68 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -24,12 +24,7 @@ use itertools::Itertools;
 use parking_lot::{Mutex, RwLock};
 use twox_hash::XxHash64;
 
-use crate::{
-    device::BufferAllocator,
-    metrics::Metrics,
-    region::{RegionId, Version},
-    ring::View,
-};
+use crate::{device::BufferAllocator, metrics::Metrics, region::RegionId, ring::View};
 
 pub type Sequence = u64;
 
@@ -40,7 +35,6 @@ pub enum Index {
     },
     Region {
         region: RegionId,
-        version: Version,
         offset: u32,
         len: u32,
         key_len: u32,
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index f8d13682..551f86d0 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -12,52 +12,101 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::Arc;
-
-use foyer_common::rate::RateLimiter;
-use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-
-use tokio::sync::broadcast;
-
 use crate::{
+    buffer::{BufferError, FlushBuffer, PositionedEntry},
+    catalog::{Catalog, Index, Item, Sequence},
     device::Device,
-    error::Result,
+    error::{Error, Result},
     metrics::Metrics,
-    region::RegionId,
     region_manager::{RegionEpItemAdapter, RegionManager},
+    ring::View,
 };
+use foyer_common::code::Key;
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use std::{any::Any, fmt::Debug, sync::Arc};
+use tokio::sync::{broadcast, mpsc};
+use tracing::Instrument;
+
+pub struct Entry {
+    /// # Safety
+    ///
+    /// `key` must be `Arc<K> where K = Flusher<K>`.
+    ///
+    /// Use `dyn Any` here to avoid contagious generic type.
+    pub key: Arc<dyn Any + Send + Sync>,
+    pub key_len: usize,
+    pub value_len: usize,
+    pub sequence: Sequence,
+
+    /// Hold a view of referenced buffer, for lookup and prevent from releasing.
+    pub view: View,
+}
+
+impl Debug for Entry {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Entry")
+            .field("key_len", &self.key_len)
+            .field("value_len", &self.value_len)
+            .field("sequence", &self.sequence)
+            .field("view", &self.view)
+            .finish()
+    }
+}
+
+impl Clone for Entry {
+    fn clone(&self) -> Self {
+        Self {
+            key: Arc::clone(&self.key),
+            view: self.view.clone(),
+            key_len: self.key_len,
+            value_len: self.value_len,
+            sequence: self.sequence,
+        }
+    }
+}
 
 #[derive(Debug)]
-pub struct Flusher<D, EP, EL>
+pub struct Flusher<K, D, EP, EL>
 where
+    K: Key,
     D: Device,
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
-    rate_limiter: Option<Arc<RateLimiter>>,
+    catalog: Arc<Catalog<K>>,
+
+    buffer: FlushBuffer<D>,
+
+    entry_rx: mpsc::UnboundedReceiver<Entry>,
 
     metrics: Arc<Metrics>,
 
     stop_rx: broadcast::Receiver<()>,
 }
 
-impl<D, EP, EL> Flusher<D, EP, EL>
+impl<K, D, EP, EL> Flusher<K, D, EP, EL>
 where
+    K: Key,
     D: Device,
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     pub fn new(
         region_manager: Arc<RegionManager<D, EP, EL>>,
-        rate_limiter: Option<Arc<RateLimiter>>,
+        catalog: Arc<Catalog<K>>,
+        device: D,
+        entry_rx: mpsc::UnboundedReceiver<Entry>,
         metrics: Arc<Metrics>,
         stop_rx: broadcast::Receiver<()>,
     ) -> Self {
+        let buffer = FlushBuffer::new(device.clone());
         Self {
             region_manager,
-            rate_limiter,
+            catalog,
+            buffer,
+            entry_rx,
             metrics,
             stop_rx,
         }
@@ -67,10 +116,16 @@ where
         loop {
             tokio::select! {
                 biased;
-                region_id = self.region_manager.dirty_regions().acquire() => {
-                    self.handle(region_id).await?;
+                entry = self.entry_rx.recv() => {
+                    let Some(entry) = entry else {
+                        self.buffer.flush().await?;
+                        tracing::info!("[flusher] exit");
+                        return Ok(());
+                    };
+                    self.handle(entry).await?;
                 }
                 _ = self.stop_rx.recv() => {
+                    self.buffer.flush().await?;
                     tracing::info!("[flusher] exit");
                     return Ok(())
                 }
@@ -78,74 +133,83 @@ where
         }
     }
 
-    async fn handle(&self, region_id: RegionId) -> Result<()> {
-        let _timer = self.metrics.slow_op_duration_flush.start_timer();
-
-        tracing::info!("[flusher] receive flush task, region: {}", region_id);
+    async fn handle(&mut self, entry: Entry) -> Result<()> {
+        let timer = self.metrics.inner_op_duration_flusher_handle.start_timer();
 
-        let region = self.region_manager.region(&region_id);
+        let old_region = self.buffer.region();
 
-        tracing::trace!("[flusher] step 1");
+        let entry = match self.buffer.write(entry).await {
+            Err(BufferError::NotEnough { entry }) => entry,
 
-        // step 1: write buffer back to device
-        let slice = region.load(.., 0).await?.unwrap();
-        let mut slice = Some(slice);
+            Ok(entries) => return self.update_catalog(entries).await,
+            Err(e) => return Err(Error::from(e)),
+        };
 
-        {
-            // wait all physical readers (from previous version) and writers done
-            let _ = region.exclusive(false, true, false).await;
+        // current region is full, rotate flush buffer region and retry
+
+        // 1. get a clean region
+        let acquire_clean_region_timer = self
+            .metrics
+            .inner_op_duration_acquire_clean_region
+            .start_timer();
+        let new_region = self
+            .region_manager
+            .clean_regions()
+            .acquire()
+            .instrument(tracing::debug_span!("acquire_clean_region"))
+            .await;
+        drop(acquire_clean_region_timer);
+
+        // 2. rotate flush buffer
+        let entries = self.buffer.rotate(new_region).await?;
+        self.update_catalog(entries).await?;
+        if let Some(old_region) = old_region {
+            self.region_manager.eviction_push(old_region);
         }
 
-        tracing::trace!("[flusher] write region {} back to device", region_id);
-
-        let mut offset = 0;
-        let len = region.device().io_size();
-        while offset < region.device().region_size() {
-            let start = offset;
-            let end = std::cmp::min(offset + len, region.device().region_size());
-
-            if let Some(limiter) = &self.rate_limiter && let Some(duration) = limiter.consume(len as f64) {
-                tokio::time::sleep(duration).await;
-            }
-            let (res, s) = region
+        self.metrics.total_bytes.add(
+            self.region_manager
+                .region(&new_region)
                 .device()
-                .write(
-                    slice.take().unwrap(),
-                    start..end,
-                    region.id(),
-                    offset as u64,
-                )
-                .await;
-            res?;
-            slice = Some(s);
-            offset += len;
-        }
-
-        drop(slice);
-
-        tracing::trace!("[flusher] step 2");
+                .region_size() as u64,
+        );
 
-        let buffer = {
-            // step 2: detach buffer
-            let mut guard = region.exclusive(false, false, true).await;
-            guard.detach_buffer()
-        };
-
-        tracing::trace!("[flusher] step 3");
-
-        // step 3: release buffer
-        self.region_manager.buffers().release(buffer);
-        self.region_manager.eviction_push(region.id());
-
-        tracing::info!("[flusher] finish flush task, region: {}", region_id);
+        // 3. retry write
+        let entries = self.buffer.write(entry).await?;
+        self.update_catalog(entries).await?;
 
-        self.metrics
-            .op_bytes_flush
-            .inc_by(region.device().region_size() as u64);
-        self.metrics
-            .total_bytes
-            .add(region.device().region_size() as u64);
+        drop(timer);
+        Ok(())
+    }
 
+    #[tracing::instrument(skip(self))]
+    async fn update_catalog(&self, entries: Vec<PositionedEntry>) -> Result<()> {
+        let timer = self.metrics.inner_op_duration_update_catalog.start_timer();
+        for PositionedEntry {
+            entry:
+                Entry {
+                    key,
+                    view,
+                    key_len,
+                    value_len,
+                    sequence,
+                },
+            region,
+            offset,
+        } in entries
+        {
+            let key = key.downcast::<K>().unwrap();
+            let index = Index::Region {
+                region,
+                offset: offset as u32,
+                len: view.aligned() as u32,
+                key_len: key_len as u32,
+                value_len: value_len as u32,
+            };
+            let item = Item::new(sequence, index);
+            self.catalog.insert(key, item);
+        }
+        drop(timer);
         Ok(())
     }
 }
diff --git a/foyer-storage/src/flusher_v2.rs b/foyer-storage/src/flusher_v2.rs
deleted file mode 100644
index 0afe6940..00000000
--- a/foyer-storage/src/flusher_v2.rs
+++ /dev/null
@@ -1,216 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use crate::{
-    buffer::{BufferError, FlushBuffer, PositionedEntry},
-    catalog::{Catalog, Index, Item, Sequence},
-    device::Device,
-    error::{Error, Result},
-    metrics::Metrics,
-    region_manager::{RegionEpItemAdapter, RegionManager},
-    ring::View,
-};
-use foyer_common::code::Key;
-use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use std::{any::Any, fmt::Debug, sync::Arc};
-use tokio::sync::{broadcast, mpsc};
-use tracing::Instrument;
-
-pub struct Entry {
-    /// # Safety
-    ///
-    /// `key` must be `Arc<K> where K = Flusher<K>`.
-    ///
-    /// Use `dyn Any` here to avoid contagious generic type.
-    pub key: Arc<dyn Any + Send + Sync>,
-    pub key_len: usize,
-    pub value_len: usize,
-    pub sequence: Sequence,
-
-    /// Hold a view of referenced buffer, for lookup and prevent from releasing.
-    pub view: View,
-}
-
-impl Debug for Entry {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Entry")
-            .field("key_len", &self.key_len)
-            .field("value_len", &self.value_len)
-            .field("sequence", &self.sequence)
-            .field("view", &self.view)
-            .finish()
-    }
-}
-
-impl Clone for Entry {
-    fn clone(&self) -> Self {
-        Self {
-            key: Arc::clone(&self.key),
-            view: self.view.clone(),
-            key_len: self.key_len,
-            value_len: self.value_len,
-            sequence: self.sequence,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Flusher<K, D, EP, EL>
-where
-    K: Key,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
-{
-    region_manager: Arc<RegionManager<D, EP, EL>>,
-
-    catalog: Arc<Catalog<K>>,
-
-    buffer: FlushBuffer<D>,
-
-    entry_rx: mpsc::UnboundedReceiver<Entry>,
-
-    metrics: Arc<Metrics>,
-
-    stop_rx: broadcast::Receiver<()>,
-}
-
-impl<K, D, EP, EL> Flusher<K, D, EP, EL>
-where
-    K: Key,
-    D: Device,
-    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-    EL: Link,
-{
-    pub fn new(
-        region_manager: Arc<RegionManager<D, EP, EL>>,
-        catalog: Arc<Catalog<K>>,
-        device: D,
-        entry_rx: mpsc::UnboundedReceiver<Entry>,
-        metrics: Arc<Metrics>,
-        stop_rx: broadcast::Receiver<()>,
-    ) -> Self {
-        let buffer = FlushBuffer::new(device.clone());
-        Self {
-            region_manager,
-            catalog,
-            buffer,
-            entry_rx,
-            metrics,
-            stop_rx,
-        }
-    }
-
-    pub async fn run(mut self) -> Result<()> {
-        loop {
-            tokio::select! {
-                biased;
-                entry = self.entry_rx.recv() => {
-                    let Some(entry) = entry else {
-                        self.buffer.flush().await?;
-                        tracing::info!("[flusher] exit");
-                        return Ok(());
-                    };
-                    self.handle(entry).await?;
-                }
-                _ = self.stop_rx.recv() => {
-                    self.buffer.flush().await?;
-                    tracing::info!("[flusher] exit");
-                    return Ok(())
-                }
-            }
-        }
-    }
-
-    async fn handle(&mut self, entry: Entry) -> Result<()> {
-        let timer = self.metrics.inner_op_duration_flusher_handle.start_timer();
-
-        let old_region = self.buffer.region();
-
-        let entry = match self.buffer.write(entry).await {
-            Err(BufferError::NotEnough { entry }) => entry,
-
-            Ok(entries) => return self.update_catalog(entries).await,
-            Err(e) => return Err(Error::from(e)),
-        };
-
-        // current region is full, rotate flush buffer region and retry
-
-        // 1. get a clean region
-        let acquire_clean_region_timer = self
-            .metrics
-            .inner_op_duration_acquire_clean_region
-            .start_timer();
-        let new_region = self
-            .region_manager
-            .clean_regions()
-            .acquire()
-            .instrument(tracing::debug_span!("acquire_clean_region"))
-            .await;
-        drop(acquire_clean_region_timer);
-
-        // 2. rotate flush buffer
-        let entries = self.buffer.rotate(new_region).await?;
-        self.update_catalog(entries).await?;
-        if let Some(old_region) = old_region {
-            self.region_manager.eviction_push(old_region);
-        }
-
-        self.metrics.total_bytes.add(
-            self.region_manager
-                .region(&new_region)
-                .device()
-                .region_size() as u64,
-        );
-
-        // 3. retry write
-        let entries = self.buffer.write(entry).await?;
-        self.update_catalog(entries).await?;
-
-        drop(timer);
-        Ok(())
-    }
-
-    #[tracing::instrument(skip(self))]
-    async fn update_catalog(&self, entries: Vec<PositionedEntry>) -> Result<()> {
-        let timer = self.metrics.inner_op_duration_update_catalog.start_timer();
-        for PositionedEntry {
-            entry:
-                Entry {
-                    key,
-                    view,
-                    key_len,
-                    value_len,
-                    sequence,
-                },
-            region,
-            offset,
-        } in entries
-        {
-            let key = key.downcast::<K>().unwrap();
-            let index = Index::Region {
-                region,
-                version: 0,
-                offset: offset as u32,
-                len: view.aligned() as u32,
-                key_len: key_len as u32,
-                value_len: value_len as u32,
-            };
-            let item = Item::new(sequence, index);
-            self.catalog.insert(key, item);
-        }
-        drop(timer);
-        Ok(())
-    }
-}
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index a5e1d33b..d0249b8a 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -40,7 +40,7 @@ use crate::{
     catalog::{Catalog, Index, Item, Sequence},
     device::Device,
     error::Result,
-    flusher_v2::{Entry, Flusher},
+    flusher::{Entry, Flusher},
     judge::Judges,
     metrics::{Metrics, METRICS},
     reclaimer::Reclaimer,
@@ -268,13 +268,10 @@ where
         ));
 
         let region_manager = Arc::new(RegionManager::new(
-            config.allocator_bits,
             buffer_count,
             device.regions(),
             config.eviction_config,
             device.clone(),
-            config.allocation_timeout,
-            metrics.clone(),
         ));
 
         let catalog = Arc::new(Catalog::new(
@@ -379,9 +376,6 @@ where
     }
 
     async fn close(&self) -> Result<()> {
-        // seal current dirty buffer and trigger flushing
-        self.seal().await;
-
         // stop and wait for flushers
         let handles = self.inner.flusher_handles.lock().drain(..).collect_vec();
         if !handles.is_empty() {
@@ -450,7 +444,6 @@ where
             // read from region
             crate::catalog::Index::Region {
                 region,
-                version,
                 offset,
                 len,
                 key_len: _,
@@ -462,7 +455,7 @@ where
                 let end = start + *len as usize;
 
                 // TODO(MrCroxx): read value only
-                let slice = match region.load(start..end, *version).await? {
+                let slice = match region.load(start..end).await? {
                     Some(slice) => slice,
                     None => {
                         // Remove index if the storage layer fails to lookup it (because of region version mismatch).
@@ -532,10 +525,6 @@ where
         bits::align_up(self.inner.device.align(), unaligned)
     }
 
-    async fn seal(&self) {
-        self.inner.region_manager.seal().await;
-    }
-
     #[tracing::instrument(skip(self))]
     async fn recover(&self, concurrency: usize) -> Result<Sequence> {
         tracing::info!("start store recovery");
@@ -963,7 +952,7 @@ where
     pub async fn open(region: Region<D>) -> Result<Option<Self>> {
         let align = region.device().align();
 
-        let slice = match region.load(..align, 0).await? {
+        let slice = match region.load(..align).await? {
             Some(slice) => slice,
             None => return Ok(None),
         };
@@ -990,11 +979,7 @@ where
             return Ok(None);
         }
 
-        let Some(slice) = self
-            .region
-            .load(self.cursor..self.cursor + align, 0)
-            .await?
-        else {
+        let Some(slice) = self.region.load(self.cursor..self.cursor + align).await? else {
             return Ok(None);
         };
 
@@ -1029,7 +1014,7 @@ where
             key
         } else {
             drop(slice);
-            let Some(s) = self.region.load(align_start..align_end, 0).await? else {
+            let Some(s) = self.region.load(align_start..align_end).await? else {
                 return Ok(None);
             };
             let rel_start = abs_start - align_start;
@@ -1044,7 +1029,6 @@ where
             header.sequence,
             Index::Region {
                 region: self.region.id(),
-                version: 0,
                 offset: self.cursor as u32,
                 len: entry_len as u32,
                 key_len: header.key_len,
@@ -1071,7 +1055,7 @@ where
         // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
         let start = *offset as usize;
         let end = start + *len as usize;
-        let Some(slice) = self.region.load(start..end, 0).await? else {
+        let Some(slice) = self.region.load(start..end).await? else {
             return Ok(None);
         };
         let kv = read_entry::<K, V>(slice.as_ref());
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 9fd4bb6f..2214f669 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -297,5 +297,6 @@ mod tests {
         handle.await.unwrap().unwrap();
 
         assert_eq!(store.lookup(&100).await.unwrap(), Some(100));
+        store.close().await.unwrap();
     }
 }
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index eaafa1be..f46a43c5 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -28,7 +28,6 @@ pub mod catalog;
 pub mod device;
 pub mod error;
 pub mod flusher;
-pub mod flusher_v2;
 pub mod generic;
 pub mod judge;
 pub mod lazy;
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 61f485dc..dd8b35e7 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -117,13 +117,11 @@ where
 
         // after drop indices and acquire exclusive lock, no writers or readers are supposed to access the region
         {
-            let guard = region.exclusive(false, false, false).await;
+            let guard = region.exclusive(false, false).await;
             tracing::trace!(
-                "[reclaimer] region {}, writers: {}, buffered readers: {}, physical readers: {}",
+                "[reclaimer] region {}, physical readers: {}",
                 region.id(),
-                guard.writers(),
-                guard.buffered_readers(),
-                guard.physical_readers()
+                guard.readers(),
             );
             drop(guard);
         }
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 91a106ef..5e48fc2f 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -27,27 +27,10 @@ use tracing::instrument;
 use crate::{
     device::{BufferAllocator, Device},
     error::Result,
-    slice::{Slice, SliceMut},
+    slice::SliceMut,
 };
 
 pub type RegionId = u32;
-/// 0 matches any version
-pub type Version = u32;
-
-#[derive(Debug)]
-pub enum AllocateResult {
-    Ok(WriteSlice),
-    NotEnough(WriteSlice),
-}
-
-impl AllocateResult {
-    pub fn unwrap(self) -> WriteSlice {
-        match self {
-            AllocateResult::Ok(slice) => slice,
-            AllocateResult::NotEnough { .. } => unreachable!(),
-        }
-    }
-}
 
 pub const REGION_MAGIC: u64 = 0x19970327;
 
@@ -73,15 +56,7 @@ pub struct RegionInner<A>
 where
     A: BufferAllocator,
 {
-    version: Version,
-
-    buffer: Option<Vec<u8, A>>,
-    len: usize,
-    capacity: usize,
-
-    writers: usize,
-    buffered_readers: usize,
-    physical_readers: usize,
+    readers: usize,
 
     #[expect(clippy::type_complexity)]
     waits: BTreeMap<(usize, usize), Vec<oneshot::Sender<Result<ReadSlice<A>>>>>,
@@ -89,18 +64,14 @@ where
 
 #[derive(Debug, Clone)]
 pub struct RegionInnerExclusiveRequire {
-    can_write: bool,
-    can_buffered_read: bool,
-    can_physical_read: bool,
+    can_read: bool,
 }
 
 impl<A: BufferAllocator> ErwLockInner for RegionInner<A> {
     type R = RegionInnerExclusiveRequire;
 
     fn is_exclusive(&self, require: &Self::R) -> bool {
-        (require.can_write || self.writers == 0)
-            && (require.can_buffered_read || self.buffered_readers == 0)
-            && (require.can_physical_read || self.physical_readers == 0)
+        require.can_read || self.readers == 0
     }
 }
 
@@ -120,29 +91,17 @@ where
 ///
 /// [`Region`] may be in one of the following states:
 ///
-/// - buffered write : append-only buffer write, written parts can be read concurrently.
-/// - buffered read  : happenes if the region is dirty with an attached dirty buffer
-/// - physical read  : happenes if the region is clean, read directly from the devie
-/// - flush          : happenes after the region dirty buffer is full, there are 2 steps when flushing
-///                    step 1 writes dirty buffer to device, must guarantee there is no writers or physical readers
-///                    step 2 detaches dirty buffer, must guarantee there is no buffer readers
-/// - reclaim        : happens after the region is evicted, must guarantee there is no writers, buffer readers or physical readers,
-///                    *or in-flight writers or readers* (verify by version)
+/// - physical write : written by flushers with append pattern
+/// - physical read  : read if entry is not in ring buffer
+/// - reclaim        : happens after the region is evicted, must guarantee there is no writers or readers,
+///                    *or in-flight writers or readers*
 impl<D> Region<D>
 where
     D: Device,
 {
     pub fn new(id: RegionId, device: D) -> Self {
         let inner = RegionInner {
-            version: 0,
-
-            buffer: None,
-            len: 0,
-            capacity: device.region_size(),
-
-            writers: 0,
-            buffered_readers: 0,
-            physical_readers: 0,
+            readers: 0,
 
             waits: BTreeMap::new(),
         };
@@ -153,57 +112,6 @@ where
         }
     }
 
-    /// If there is enough buffer, return `AllocateResult::Ok(slice)``.
-    /// Else, return `AllocateResult::NotEnough(slice)`. `slice` is the remaining buffer.
-    #[tracing::instrument(skip(self))]
-    pub fn allocate(&self, size: usize) -> AllocateResult {
-        let cleanup = {
-            let inner = self.inner.clone();
-            let f = move || {
-                let mut guard = inner.write();
-                guard.writers -= 1;
-            };
-            Box::new(f)
-        };
-
-        let mut inner = self.inner.write();
-
-        inner.writers += 1;
-        let version = inner.version;
-        let offset = inner.len;
-        let region_id = self.id;
-
-        if inner.len + size > inner.capacity {
-            inner.len = self.device.region_size();
-
-            let buffer = inner.buffer.as_mut().unwrap();
-            let slice = unsafe { SliceMut::new(&mut buffer[offset..]) };
-
-            let slice = WriteSlice {
-                slice,
-                region_id,
-                version,
-                offset,
-                cleanup: Some(cleanup),
-            };
-            AllocateResult::NotEnough(slice)
-        } else {
-            inner.len += size;
-
-            let buffer = inner.buffer.as_mut().unwrap();
-            let slice = unsafe { SliceMut::new(&mut buffer[offset..offset + size]) };
-
-            let slice = WriteSlice {
-                slice,
-                region_id,
-                version,
-                offset,
-                cleanup: Some(cleanup),
-            };
-            AllocateResult::Ok(slice)
-        }
-    }
-
     /// Load region data into a [`ReadSlice`].
     ///
     /// Data may be loaded ether from physical device or from dirty buffer.
@@ -215,7 +123,6 @@ where
     pub async fn load(
         &self,
         range: impl RangeBounds<usize>,
-        version: Version,
     ) -> Result<Option<ReadSlice<D::IoBufferAllocator>>> {
         let start = match range.start_bound() {
             std::ops::Bound::Included(i) => *i,
@@ -228,38 +135,10 @@ where
             std::ops::Bound::Unbounded => self.device.region_size(),
         };
 
-        // case 1: read from dirty buffer
-
-        // restrict guard lifetime
         let rx = {
             let mut inner = self.inner.write();
 
-            if version != 0 && version != inner.version {
-                return Ok(None);
-            }
-
-            // if buffer attached, buffered read
-
-            if inner.buffer.is_some() {
-                inner.buffered_readers += 1;
-                let allocator = inner.buffer.as_ref().unwrap().allocator().clone();
-                let slice = unsafe { Slice::new(&inner.buffer.as_ref().unwrap()[start..end]) };
-                let cleanup = {
-                    let inner = self.inner.clone();
-                    let f = move || {
-                        let mut guard = inner.write();
-                        guard.buffered_readers -= 1;
-                    };
-                    Box::new(f)
-                };
-                return Ok(Some(ReadSlice::Slice {
-                    slice,
-                    allocator: Some(allocator),
-                    cleanup: Some(cleanup),
-                }));
-            }
-
-            // case 2: join wait map if exists
+            // join wait map if exists
             let rx = match inner.waits.entry((start, end)) {
                 Entry::Vacant(v) => {
                     v.insert(vec![]);
@@ -272,18 +151,18 @@ where
                 }
             };
 
-            inner.physical_readers += 1;
+            inner.readers += 1;
             drop(inner);
 
             rx
         };
 
-        // case 3: wait for result
+        // wait for result if joined into wait map
         if let Some(rx) = rx {
             return rx.await.map_err(anyhow::Error::from)?.map(Some);
         }
 
-        // case 4: read from device
+        // otherwise, read from device
         let region = self.id;
         let mut buf = self.device.io_buffer(end - start, end - start);
 
@@ -291,7 +170,7 @@ where
         while start + offset < end {
             let len = std::cmp::min(self.device.io_size(), end - start - offset);
             tracing::trace!(
-                "physical read region {} [{}..{}]",
+                "read region {} [{}..{}]",
                 region,
                 start + offset,
                 start + offset + len
@@ -306,14 +185,14 @@ where
                 Err(e) => {
                     let mut inner = self.inner.write();
                     self.cleanup(&mut inner, start, end)?;
-                    inner.physical_readers -= 1;
+                    inner.readers -= 1;
                     return Err(e.into());
                 }
             };
             if read != len {
                 let mut inner = self.inner.write();
                 self.cleanup(&mut inner, start, end)?;
-                inner.physical_readers -= 1;
+                inner.readers -= 1;
                 return Ok(None);
             }
             offset += len;
@@ -324,7 +203,7 @@ where
             let inner = self.inner.clone();
             let f = move || {
                 let mut guard = inner.write();
-                guard.physical_readers -= 1;
+                guard.readers -= 1;
             };
             Box::new(f)
         };
@@ -332,7 +211,7 @@ where
         if let Some(txs) = self.inner.write().waits.remove(&(start, end)) {
             // TODO: handle error !!!!!!!!!!!
             for tx in txs {
-                tx.send(Ok(ReadSlice::Shared {
+                tx.send(Ok(ReadSlice {
                     buf: buf.clone(),
                     cleanup: Some(cleanup.clone()),
                 }))
@@ -340,51 +219,20 @@ where
             }
         }
 
-        Ok(Some(ReadSlice::Shared {
+        Ok(Some(ReadSlice {
             buf,
             cleanup: Some(cleanup),
         }))
     }
 
-    pub async fn attach_buffer(&self, buf: Vec<u8, D::IoBufferAllocator>) {
-        let mut inner = self.inner.write();
-
-        assert_eq!(inner.writers, 0);
-        assert_eq!(inner.buffered_readers, 0);
-
-        inner.attach_buffer(buf);
-        let buffer = inner.buffer.as_deref_mut().unwrap();
-        let header = RegionHeader {
-            magic: REGION_MAGIC,
-        };
-        header.write(buffer);
-        inner.len = self.device.align();
-    }
-
-    pub async fn detach_buffer(&self) -> Vec<u8, D::IoBufferAllocator> {
-        let mut inner = self.inner.write();
-
-        inner.detach_buffer()
-    }
-
-    pub async fn has_buffer(&self) -> bool {
-        let inner = self.inner.read();
-        inner.has_buffer()
-    }
-
     #[instrument(skip(self))]
     pub async fn exclusive(
         &self,
         can_write: bool,
-        can_buffered_read: bool,
-        can_physical_read: bool,
+        can_read: bool,
     ) -> ArcRwLockWriteGuard<RawRwLock, RegionInner<D::IoBufferAllocator>> {
         self.inner
-            .exclusive(&RegionInnerExclusiveRequire {
-                can_write,
-                can_buffered_read,
-                can_physical_read,
-            })
+            .exclusive(&RegionInnerExclusiveRequire { can_read })
             .await
     }
 
@@ -396,17 +244,6 @@ where
         &self.device
     }
 
-    pub async fn version(&self) -> Version {
-        self.inner.read().version
-    }
-
-    pub async fn advance(&self) -> Version {
-        let mut inner = self.inner.write();
-        let res = inner.version;
-        inner.version += 1;
-        res
-    }
-
     /// Cleanup waits.
     fn cleanup(
         &self,
@@ -415,7 +252,7 @@ where
         end: usize,
     ) -> Result<()> {
         if let Some(txs) = guard.waits.remove(&(start, end)) {
-            guard.writers -= txs.len();
+            guard.readers -= txs.len();
             for tx in txs {
                 tx.send(Err(anyhow::anyhow!("cancelled by previous error").into()))
                     .map_err(|_| anyhow::anyhow!("fail to cleanup waits"))?;
@@ -429,32 +266,8 @@ impl<A> RegionInner<A>
 where
     A: BufferAllocator,
 {
-    pub fn attach_buffer(&mut self, buf: Vec<u8, A>) {
-        assert!(self.buffer.is_none());
-        assert_eq!(buf.len(), buf.capacity());
-        assert_eq!(buf.capacity(), self.capacity);
-        self.buffer = Some(buf);
-        self.len = 0;
-    }
-
-    pub fn detach_buffer(&mut self) -> Vec<u8, A> {
-        self.buffer.take().unwrap()
-    }
-
-    pub fn has_buffer(&self) -> bool {
-        self.buffer.is_some()
-    }
-
-    pub fn writers(&self) -> usize {
-        self.writers
-    }
-
-    pub fn buffered_readers(&self) -> usize {
-        self.buffered_readers
-    }
-
-    pub fn physical_readers(&self) -> usize {
-        self.physical_readers
+    pub fn readers(&self) -> usize {
+        self.readers
     }
 }
 
@@ -462,80 +275,12 @@ where
 
 pub trait CleanupFn = FnOnce() + Send + Sync + 'static;
 
-pub struct WriteSlice {
-    slice: SliceMut,
-    region_id: RegionId,
-    version: Version,
-    offset: usize,
-    cleanup: Option<Box<dyn CleanupFn>>,
-}
-
-impl Debug for WriteSlice {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("WriteSlice")
-            .field("slice", &self.slice)
-            .field("region_id", &self.region_id)
-            .field("version", &self.version)
-            .field("offset", &self.offset)
-            .finish()
-    }
-}
-
-impl WriteSlice {
-    pub fn region_id(&self) -> RegionId {
-        self.region_id
-    }
-
-    pub fn version(&self) -> Version {
-        self.version
-    }
-
-    pub fn offset(&self) -> usize {
-        self.offset
-    }
-
-    pub fn len(&self) -> usize {
-        self.slice.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-}
-
-impl AsRef<[u8]> for WriteSlice {
-    fn as_ref(&self) -> &[u8] {
-        self.slice.as_ref()
-    }
-}
-
-impl AsMut<[u8]> for WriteSlice {
-    fn as_mut(&mut self) -> &mut [u8] {
-        self.slice.as_mut()
-    }
-}
-
-impl Drop for WriteSlice {
-    fn drop(&mut self) {
-        if let Some(f) = self.cleanup.take() {
-            f()
-        }
-    }
-}
-
-pub enum ReadSlice<A>
+pub struct ReadSlice<A>
 where
     A: BufferAllocator,
 {
-    Slice {
-        slice: Slice,
-        allocator: Option<A>,
-        cleanup: Option<Box<dyn CleanupFn>>,
-    },
-    Shared {
-        buf: Arc<Vec<u8, A>>,
-        cleanup: Option<Box<dyn CleanupFn>>,
-    },
+    buf: Arc<Vec<u8, A>>,
+    cleanup: Option<Box<dyn CleanupFn>>,
 }
 
 impl<A> Debug for ReadSlice<A>
@@ -543,19 +288,9 @@ where
     A: BufferAllocator,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::Slice {
-                slice, allocator, ..
-            } => f
-                .debug_struct("ReadSlice::Slice")
-                .field("slice", slice)
-                .field("allocator", allocator)
-                .finish(),
-            Self::Shared { buf, .. } => f
-                .debug_struct("ReadSlice::Shared")
-                .field("buf", buf)
-                .finish(),
-        }
+        f.debug_struct("ReadSlice")
+            .field("len", &self.buf.len())
+            .finish()
     }
 }
 
@@ -564,10 +299,7 @@ where
     A: BufferAllocator,
 {
     pub fn len(&self) -> usize {
-        match self {
-            Self::Slice { slice, .. } => slice.len(),
-            Self::Shared { buf, .. } => buf.len(),
-        }
+        self.buf.len()
     }
 
     pub fn is_empty(&self) -> bool {
@@ -580,10 +312,7 @@ where
     A: BufferAllocator,
 {
     fn as_ref(&self) -> &[u8] {
-        match self {
-            Self::Slice { slice, .. } => slice.as_ref(),
-            Self::Shared { buf, .. } => buf.as_ref(),
-        }
+        self.buf.as_ref()
     }
 }
 
@@ -592,10 +321,7 @@ where
     A: BufferAllocator,
 {
     fn drop(&mut self) {
-        if let Some(f) = match self {
-            ReadSlice::Slice { cleanup, .. } => cleanup.take(),
-            ReadSlice::Shared { cleanup, .. } => cleanup.take(),
-        } {
+        if let Some(f) = self.cleanup.take() {
             f();
         }
     }
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index a80bf89d..057b760a 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -12,13 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc,
-    },
-    time::Duration,
-};
+use std::sync::Arc;
 
 use foyer_common::queue::AsyncQueue;
 use foyer_intrusive::{
@@ -26,15 +20,12 @@ use foyer_intrusive::{
     eviction::{EvictionPolicy, EvictionPolicyExt},
     intrusive_adapter, key_adapter,
 };
-use itertools::Itertools;
+
 use parking_lot::RwLock;
-use tokio::sync::Mutex as AsyncMutex;
-use tracing::Instrument;
 
 use crate::{
     device::Device,
-    metrics::Metrics,
-    region::{AllocateResult, Region, RegionId, WriteSlice},
+    region::{Region, RegionId},
 };
 
 #[derive(Debug)]
@@ -49,11 +40,6 @@ where
 intrusive_adapter! { pub RegionEpItemAdapter<L> = Arc<RegionEpItem<L>>: RegionEpItem<L> { link: L } where L: Link }
 key_adapter! { RegionEpItemAdapter<L> = RegionEpItem<L> { id: RegionId } where L: Link }
 
-/// Manager of regions and buffer pools.
-///
-/// # Region Lifetime
-///
-/// `clean` ==(allocate)=> `dirty` ==(flush)=> `evictable` ==(reclaim)=> `clean`
 #[derive(Debug)]
 pub struct RegionManager<D, EP, EL>
 where
@@ -61,28 +47,14 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    allocators: Vec<AsyncMutex<Option<Region<D>>>>,
-    allocator_bits: usize,
-    allocated: AtomicUsize,
-
-    /// Buffer pool for dirty buffers.
-    buffers: AsyncQueue<Vec<u8, D::IoBufferAllocator>>,
-
     /// Empty regions.
     clean_regions: AsyncQueue<RegionId>,
 
-    /// Regions with dirty buffer waiting for flushing.
-    dirty_regions: AsyncQueue<RegionId>,
-
     regions: Vec<Region<D>>,
     items: Vec<Arc<RegionEpItem<EL>>>,
 
     /// Eviction policy.
     eviction: RwLock<EP>,
-
-    allocation_timeout: Duration,
-
-    metrics: Arc<Metrics>,
 }
 
 impl<D, EP, EL> RegionManager<D, EP, EL>
@@ -92,13 +64,10 @@ where
     EL: Link,
 {
     pub fn new(
-        allocator_bits: usize,
         buffer_count: usize,
         region_count: usize,
         eviction_config: EP::Config,
         device: D,
-        allocation_timeout: Duration,
-        metrics: Arc<Metrics>,
     ) -> Self {
         let buffers = AsyncQueue::new();
         for _ in 0..buffer_count {
@@ -109,7 +78,6 @@ where
 
         let eviction = EP::new(eviction_config);
         let clean_regions = AsyncQueue::new();
-        let dirty_regions = AsyncQueue::new();
 
         let mut regions = Vec::with_capacity(region_count);
         let mut items = Vec::with_capacity(region_count);
@@ -125,99 +93,11 @@ where
             items.push(item);
         }
 
-        let allocators = (0..(1 << allocator_bits))
-            .map(|_| AsyncMutex::new(None))
-            .collect_vec();
-
         Self {
-            allocators,
-            allocated: AtomicUsize::new(0),
-            allocator_bits,
-            buffers,
             clean_regions,
-            dirty_regions,
             regions,
             items,
             eviction: RwLock::new(eviction),
-            allocation_timeout,
-            metrics,
-        }
-    }
-
-    #[tracing::instrument(skip(self))]
-    pub async fn allocate(&self, size: usize, must_allocate: bool) -> Option<WriteSlice> {
-        let allocated = self.allocated.fetch_add(1, Ordering::Relaxed);
-        let index = allocated & ((1 << self.allocator_bits) - 1);
-        let allocator = &self.allocators[index];
-
-        let mut current = if must_allocate {
-            allocator.lock().await
-        } else {
-            match tokio::time::timeout(self.allocation_timeout, allocator.lock()).await {
-                Ok(current) => current,
-                Err(_) => return None,
-            }
-        };
-
-        loop {
-            if let Some(region) = current.as_ref() {
-                match region.allocate(size) {
-                    AllocateResult::Ok(slice) => return Some(slice),
-                    AllocateResult::NotEnough { .. } => {
-                        self.dirty_regions.release(region.id());
-                        *current = None;
-                    }
-                }
-            }
-
-            assert!(current.is_none());
-
-            // Wait a clean region to be released.
-            let region_id = {
-                let timer = self
-                    .metrics
-                    .inner_op_duration_acquire_clean_region
-                    .start_timer();
-                let region_id = self
-                    .clean_regions
-                    .acquire()
-                    .instrument(tracing::debug_span!("acquire_clean_region"))
-                    .await;
-                drop(timer);
-                region_id
-            };
-
-            tracing::info!("allocator {} switch to clean region: {}", index, region_id);
-
-            let region = self.region(&region_id);
-            region.advance().await;
-
-            let buffer = {
-                let timer = self
-                    .metrics
-                    .inner_op_duration_acquire_clean_buffer
-                    .start_timer();
-                let buffer = self
-                    .buffers
-                    .acquire()
-                    .instrument(tracing::debug_span!("acquire_clean_buffer"))
-                    .await;
-                drop(timer);
-                buffer
-            };
-            region.attach_buffer(buffer).await;
-
-            *current = Some(region.clone());
-        }
-    }
-
-    pub async fn seal(&self) {
-        for allocator in self.allocators.iter() {
-            let mut guard = allocator.lock().await;
-            if let Some(region) = guard.as_ref() {
-                self.dirty_regions.release(region.id());
-            }
-            *guard = None;
         }
     }
 
@@ -234,18 +114,10 @@ where
         }
     }
 
-    pub fn buffers(&self) -> &AsyncQueue<Vec<u8, D::IoBufferAllocator>> {
-        &self.buffers
-    }
-
     pub fn clean_regions(&self) -> &AsyncQueue<RegionId> {
         &self.clean_regions
     }
 
-    pub fn dirty_regions(&self) -> &AsyncQueue<RegionId> {
-        &self.dirty_regions
-    }
-
     pub fn eviction_push(&self, region_id: RegionId) {
         self.eviction
             .write()

From a2b41d34747b7a33be900d0780b3904a1b30284a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 31 Oct 2023 09:26:28 +0800
Subject: [PATCH 159/261] refactor: remove ErwLock usage (#195)

* chore: remove unnecessary metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: remove erwloc usagek

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: introduce RegionView

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: remove reader counter

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refine load interface

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/buffer.rs    |   4 +-
 foyer-storage/src/catalog.rs   |  31 ++---
 foyer-storage/src/flusher.rs   |  15 +--
 foyer-storage/src/generic.rs   |  50 ++++----
 foyer-storage/src/metrics.rs   |   5 -
 foyer-storage/src/reclaimer.rs |  24 ++--
 foyer-storage/src/region.rs    | 215 +++++++++++++++------------------
 foyer-storage/src/ring.rs      |  48 ++++----
 8 files changed, 187 insertions(+), 205 deletions(-)

diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 5027cb60..aaec2741 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -218,12 +218,12 @@ mod tests {
 
     use crate::{
         device::fs::{FsDevice, FsDeviceConfig},
-        ring::{RingBuffer, View},
+        ring::{RingBuffer, RingBufferView},
     };
 
     use super::*;
 
-    fn ent(view: View) -> Entry {
+    fn ent(view: RingBufferView) -> Entry {
         Entry {
             key: Arc::new(()),
             view,
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 92dd5c68..b88d7e5e 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -24,22 +24,19 @@ use itertools::Itertools;
 use parking_lot::{Mutex, RwLock};
 use twox_hash::XxHash64;
 
-use crate::{device::BufferAllocator, metrics::Metrics, region::RegionId, ring::View};
+use crate::{
+    device::BufferAllocator,
+    metrics::Metrics,
+    region::{RegionId, RegionView},
+    ring::RingBufferView,
+};
 
 pub type Sequence = u64;
 
 #[derive(Debug, Clone)]
 pub enum Index {
-    RingBuffer {
-        view: View,
-    },
-    Region {
-        region: RegionId,
-        offset: u32,
-        len: u32,
-        key_len: u32,
-        value_len: u32,
-    },
+    RingBuffer { view: RingBufferView },
+    Region { view: RegionView },
 }
 
 #[derive(Debug, Clone)]
@@ -66,6 +63,10 @@ impl Item {
     pub fn index(&self) -> &Index {
         &self.index
     }
+
+    pub fn consume(self) -> (Sequence, Index) {
+        (self.sequence, self.index)
+    }
 }
 
 #[derive(Debug)]
@@ -108,8 +109,8 @@ where
     pub fn insert(&self, key: Arc<K>, mut item: Item) {
         // TODO(MrCroxx): compare sequence.
 
-        if let Index::Region { region, .. } = item.index {
-            self.regions[region as usize]
+        if let Index::Region { view } = &item.index {
+            self.regions[*view.id() as usize]
                 .lock()
                 .insert(key.clone(), item.sequence);
         };
@@ -134,8 +135,8 @@ where
     pub fn remove(&self, key: &K) -> Option<Item> {
         let shard = self.shard(key);
         let info: Option<Item> = self.items[shard].write().remove(key);
-        if let Some(info) = &info && let Index::Region { region,..} = info.index {
-            self.regions[region as usize].lock().remove(key);
+        if let Some(info) = &info && let Index::Region { view} = &info.index {
+            self.regions[*view.id() as usize].lock().remove(key);
         }
         info
     }
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 551f86d0..f404d3ee 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -19,7 +19,7 @@ use crate::{
     error::{Error, Result},
     metrics::Metrics,
     region_manager::{RegionEpItemAdapter, RegionManager},
-    ring::View,
+    ring::RingBufferView,
 };
 use foyer_common::code::Key;
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
@@ -39,7 +39,7 @@ pub struct Entry {
     pub sequence: Sequence,
 
     /// Hold a view of referenced buffer, for lookup and prevent from releasing.
-    pub view: View,
+    pub view: RingBufferView,
 }
 
 impl Debug for Entry {
@@ -200,11 +200,12 @@ where
         {
             let key = key.downcast::<K>().unwrap();
             let index = Index::Region {
-                region,
-                offset: offset as u32,
-                len: view.aligned() as u32,
-                key_len: key_len as u32,
-                value_len: value_len as u32,
+                view: self.region_manager.region(&region).view(
+                    offset as u32,
+                    view.aligned() as u32,
+                    key_len as u32,
+                    value_len as u32,
+                ),
             };
             let item = Item::new(sequence, index);
             self.catalog.insert(key, item);
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index d0249b8a..1ef39808 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -412,8 +412,8 @@ where
     async fn lookup(&self, key: &K) -> Result<Option<V>> {
         let now = Instant::now();
 
-        let item = match self.inner.catalog.lookup(key) {
-            Some(item) => item,
+        let (_sequence, index) = match self.inner.catalog.lookup(key) {
+            Some(item) => item.consume(),
             None => {
                 self.inner
                     .metrics
@@ -423,7 +423,7 @@ where
             }
         };
 
-        match item.index() {
+        match index {
             crate::catalog::Index::RingBuffer { view } => {
                 let res = match read_entry::<K, V>(view.as_ref()) {
                     Some((_key, value)) => Ok(Some(value)),
@@ -442,20 +442,14 @@ where
                 res
             }
             // read from region
-            crate::catalog::Index::Region {
-                region,
-                offset,
-                len,
-                key_len: _,
-                value_len: _,
-            } => {
+            crate::catalog::Index::Region { view } => {
+                let region = view.id();
+
                 self.inner.region_manager.record_access(region);
                 let region = self.inner.region_manager.region(region);
-                let start = *offset as usize;
-                let end = start + *len as usize;
 
                 // TODO(MrCroxx): read value only
-                let slice = match region.load(start..end).await? {
+                let slice = match region.load(view).await? {
                     Some(slice) => slice,
                     None => {
                         // Remove index if the storage layer fails to lookup it (because of region version mismatch).
@@ -481,7 +475,6 @@ where
                         Ok(None)
                     }
                 };
-                drop(slice);
 
                 self.inner
                     .metrics
@@ -952,7 +945,7 @@ where
     pub async fn open(region: Region<D>) -> Result<Option<Self>> {
         let align = region.device().align();
 
-        let slice = match region.load(..align).await? {
+        let slice = match region.load_range(..align).await? {
             Some(slice) => slice,
             None => return Ok(None),
         };
@@ -979,7 +972,11 @@ where
             return Ok(None);
         }
 
-        let Some(slice) = self.region.load(self.cursor..self.cursor + align).await? else {
+        let Some(slice) = self
+            .region
+            .load_range(self.cursor..self.cursor + align)
+            .await?
+        else {
             return Ok(None);
         };
 
@@ -1014,7 +1011,7 @@ where
             key
         } else {
             drop(slice);
-            let Some(s) = self.region.load(align_start..align_end).await? else {
+            let Some(s) = self.region.load_range(align_start..align_end).await? else {
                 return Ok(None);
             };
             let rel_start = abs_start - align_start;
@@ -1028,11 +1025,12 @@ where
         let info = Item::new(
             header.sequence,
             Index::Region {
-                region: self.region.id(),
-                offset: self.cursor as u32,
-                len: entry_len as u32,
-                key_len: header.key_len,
-                value_len: header.value_len,
+                view: self.region.view(
+                    self.cursor as u32,
+                    entry_len as u32,
+                    header.key_len,
+                    header.value_len,
+                ),
             },
         );
 
@@ -1048,14 +1046,14 @@ where
             Err(e) => return Err(e),
         };
 
-        let Index::Region { offset, len, .. } = item.index() else {
+        let Index::Region { view } = item.index() else {
             unreachable!("kv loaded from region must have index of region")
         };
 
         // TODO(MrCroxx): Optimize if all key, value and footer are in the same read block.
-        let start = *offset as usize;
-        let end = start + *len as usize;
-        let Some(slice) = self.region.load(start..end).await? else {
+        let start = *view.offset() as usize;
+        let end = start + *view.len() as usize;
+        let Some(slice) = self.region.load_range(start..end).await? else {
             return Ok(None);
         };
         let kv = read_entry::<K, V>(slice.as_ref());
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index fdf34a44..f6ea99fe 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -154,7 +154,6 @@ pub struct Metrics {
     pub op_duration_lookup_hit: Histogram,
     pub op_duration_lookup_miss: Histogram,
     pub op_duration_remove: Histogram,
-    pub slow_op_duration_flush: Histogram,
     pub slow_op_duration_reclaim: Histogram,
 
     pub op_bytes_insert: IntCounter,
@@ -193,9 +192,6 @@ impl Metrics {
             .op_duration
             .with_label_values(&[foyer, "lookup", "miss"]);
         let op_duration_remove = global.op_duration.with_label_values(&[foyer, "remove", ""]);
-        let slow_op_duration_flush = global
-            .slow_op_duration
-            .with_label_values(&[foyer, "flush", ""]);
         let slow_op_duration_reclaim = global
             .slow_op_duration
             .with_label_values(&[foyer, "reclaim", ""]);
@@ -244,7 +240,6 @@ impl Metrics {
             op_duration_lookup_hit,
             op_duration_lookup_miss,
             op_duration_remove,
-            slow_op_duration_flush,
             slow_op_duration_reclaim,
 
             op_bytes_insert,
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index dd8b35e7..7c65b163 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -12,7 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{sync::Arc, time::Duration};
+use std::{
+    sync::{atomic::Ordering, Arc},
+    time::Duration,
+};
 
 use crate::{
     device::Device,
@@ -113,17 +116,18 @@ where
         let region = self.region_manager.region(&region_id);
 
         // step 1: drop indices
-        let _indices = self.store.catalog().take_region(&region_id);
+        let indices = self.store.catalog().take_region(&region_id);
+
+        // Must guarantee there is no following reads on the region to be reclaim.
+        // Which means there is no unfinished reader or reader who holds index and prepare to read.
 
-        // after drop indices and acquire exclusive lock, no writers or readers are supposed to access the region
+        // wait unfinished readers
         {
-            let guard = region.exclusive(false, false).await;
-            tracing::trace!(
-                "[reclaimer] region {}, physical readers: {}",
-                region.id(),
-                guard.readers(),
-            );
-            drop(guard);
+            // only each `indices` holds one ref
+            while region.refs().load(Ordering::SeqCst) > indices.len() {
+                println!("refs: {}", region.refs().load(Ordering::SeqCst));
+                tokio::time::sleep(Duration::from_millis(1)).await;
+            }
         }
 
         // step 2: do reinsertion
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 5e48fc2f..4320a8a7 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -13,16 +13,17 @@
 //  limitations under the License.
 
 use bytes::{Buf, BufMut};
-use foyer_common::erwlock::{ErwLock, ErwLockInner};
-use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock, RwLockWriteGuard};
+use parking_lot::{Mutex, MutexGuard};
 use std::{
     collections::btree_map::{BTreeMap, Entry},
     fmt::Debug,
     ops::RangeBounds,
-    sync::Arc,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
 };
 use tokio::sync::oneshot;
-use tracing::instrument;
 
 use crate::{
     device::{BufferAllocator, Device},
@@ -56,23 +57,8 @@ pub struct RegionInner<A>
 where
     A: BufferAllocator,
 {
-    readers: usize,
-
     #[expect(clippy::type_complexity)]
-    waits: BTreeMap<(usize, usize), Vec<oneshot::Sender<Result<ReadSlice<A>>>>>,
-}
-
-#[derive(Debug, Clone)]
-pub struct RegionInnerExclusiveRequire {
-    can_read: bool,
-}
-
-impl<A: BufferAllocator> ErwLockInner for RegionInner<A> {
-    type R = RegionInnerExclusiveRequire;
-
-    fn is_exclusive(&self, require: &Self::R) -> bool {
-        require.can_read || self.readers == 0
-    }
+    waits: BTreeMap<(usize, usize), Vec<oneshot::Sender<Result<Arc<Vec<u8, A>>>>>>,
 }
 
 #[derive(Debug, Clone)]
@@ -82,9 +68,11 @@ where
 {
     id: RegionId,
 
-    inner: ErwLock<RegionInner<D::IoBufferAllocator>>,
+    inner: Arc<Mutex<RegionInner<D::IoBufferAllocator>>>,
 
     device: D,
+
+    refs: Arc<AtomicUsize>,
 }
 
 /// [`Region`] represents a contiguous aligned range on device and its optional dirty buffer.
@@ -101,29 +89,54 @@ where
 {
     pub fn new(id: RegionId, device: D) -> Self {
         let inner = RegionInner {
-            readers: 0,
-
             waits: BTreeMap::new(),
         };
         Self {
             id,
-            inner: ErwLock::new(inner),
+            inner: Arc::new(Mutex::new(inner)),
             device,
+            refs: Arc::new(AtomicUsize::default()),
         }
     }
 
-    /// Load region data into a [`ReadSlice`].
-    ///
-    /// Data may be loaded ether from physical device or from dirty buffer.
-    ///
-    /// Use version `0` to skip version check.
-    ///
-    /// Returns `None` if verion mismatch or given range cannot be fully filled.
-    #[tracing::instrument(skip(self, range), fields(start, end))]
+    pub fn view(&self, offset: u32, len: u32, key_len: u32, value_len: u32) -> RegionView {
+        self.refs.fetch_add(1, Ordering::SeqCst);
+        RegionView {
+            id: self.id,
+            offset,
+            len,
+            key_len,
+            value_len,
+            refs: Arc::clone(&self.refs),
+        }
+    }
+
+    pub fn refs(&self) -> &Arc<AtomicUsize> {
+        &self.refs
+    }
+
+    /// Load region data by view from device.
+    #[expect(clippy::type_complexity)]
+    #[tracing::instrument(skip(self, view))]
     pub async fn load(
+        &self,
+        view: RegionView,
+    ) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
+        let res = self
+            .load_range(view.offset as usize..view.offset as usize + view.len as usize)
+            .await;
+        // drop view after load finish
+        drop(view);
+        res
+    }
+
+    /// Load region data with given `range` from device.
+    #[expect(clippy::type_complexity)]
+    #[tracing::instrument(skip(self, range), fields(start, end))]
+    pub async fn load_range(
         &self,
         range: impl RangeBounds<usize>,
-    ) -> Result<Option<ReadSlice<D::IoBufferAllocator>>> {
+    ) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
         let start = match range.start_bound() {
             std::ops::Bound::Included(i) => *i,
             std::ops::Bound::Excluded(i) => *i + 1,
@@ -136,7 +149,7 @@ where
         };
 
         let rx = {
-            let mut inner = self.inner.write();
+            let mut inner = self.inner.lock();
 
             // join wait map if exists
             let rx = match inner.waits.entry((start, end)) {
@@ -151,7 +164,6 @@ where
                 }
             };
 
-            inner.readers += 1;
             drop(inner);
 
             rx
@@ -183,57 +195,30 @@ where
             let read = match res {
                 Ok(bytes) => bytes,
                 Err(e) => {
-                    let mut inner = self.inner.write();
+                    let mut inner = self.inner.lock();
                     self.cleanup(&mut inner, start, end)?;
-                    inner.readers -= 1;
                     return Err(e.into());
                 }
             };
             if read != len {
-                let mut inner = self.inner.write();
+                let mut inner = self.inner.lock();
                 self.cleanup(&mut inner, start, end)?;
-                inner.readers -= 1;
+                // TODO(MrCroxx): return err?
                 return Ok(None);
             }
             offset += len;
         }
         let buf = Arc::new(buf);
 
-        let cleanup = {
-            let inner = self.inner.clone();
-            let f = move || {
-                let mut guard = inner.write();
-                guard.readers -= 1;
-            };
-            Box::new(f)
-        };
-
-        if let Some(txs) = self.inner.write().waits.remove(&(start, end)) {
+        if let Some(txs) = self.inner.lock().waits.remove(&(start, end)) {
             // TODO: handle error !!!!!!!!!!!
             for tx in txs {
-                tx.send(Ok(ReadSlice {
-                    buf: buf.clone(),
-                    cleanup: Some(cleanup.clone()),
-                }))
-                .map_err(|_| anyhow::anyhow!("fail to send load result"))?;
+                tx.send(Ok(buf.clone()))
+                    .map_err(|_| anyhow::anyhow!("fail to send load result"))?;
             }
         }
 
-        Ok(Some(ReadSlice {
-            buf,
-            cleanup: Some(cleanup),
-        }))
-    }
-
-    #[instrument(skip(self))]
-    pub async fn exclusive(
-        &self,
-        can_write: bool,
-        can_read: bool,
-    ) -> ArcRwLockWriteGuard<RawRwLock, RegionInner<D::IoBufferAllocator>> {
-        self.inner
-            .exclusive(&RegionInnerExclusiveRequire { can_read })
-            .await
+        Ok(Some(buf))
     }
 
     pub fn id(&self) -> RegionId {
@@ -247,12 +232,11 @@ where
     /// Cleanup waits.
     fn cleanup(
         &self,
-        guard: &mut RwLockWriteGuard<'_, RegionInner<D::IoBufferAllocator>>,
+        guard: &mut MutexGuard<'_, RegionInner<D::IoBufferAllocator>>,
         start: usize,
         end: usize,
     ) -> Result<()> {
         if let Some(txs) = guard.waits.remove(&(start, end)) {
-            guard.readers -= txs.len();
             for tx in txs {
                 tx.send(Err(anyhow::anyhow!("cancelled by previous error").into()))
                     .map_err(|_| anyhow::anyhow!("fail to cleanup waits"))?;
@@ -262,67 +246,62 @@ where
     }
 }
 
-impl<A> RegionInner<A>
-where
-    A: BufferAllocator,
-{
-    pub fn readers(&self) -> usize {
-        self.readers
-    }
-}
-
 // read & write slice
 
 pub trait CleanupFn = FnOnce() + Send + Sync + 'static;
 
-pub struct ReadSlice<A>
-where
-    A: BufferAllocator,
-{
-    buf: Arc<Vec<u8, A>>,
-    cleanup: Option<Box<dyn CleanupFn>>,
+#[derive(Debug)]
+pub struct RegionView {
+    id: RegionId,
+    offset: u32,
+    len: u32,
+    key_len: u32,
+    value_len: u32,
+    refs: Arc<AtomicUsize>,
 }
 
-impl<A> Debug for ReadSlice<A>
-where
-    A: BufferAllocator,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("ReadSlice")
-            .field("len", &self.buf.len())
-            .finish()
+impl Clone for RegionView {
+    fn clone(&self) -> Self {
+        self.refs.fetch_add(1, Ordering::SeqCst);
+        Self {
+            id: self.id,
+            offset: self.offset,
+            len: self.len,
+            key_len: self.key_len,
+            value_len: self.value_len,
+            refs: Arc::clone(&self.refs),
+        }
     }
 }
 
-impl<A> ReadSlice<A>
-where
-    A: BufferAllocator,
-{
-    pub fn len(&self) -> usize {
-        self.buf.len()
+impl Drop for RegionView {
+    fn drop(&mut self) {
+        self.refs.fetch_sub(1, Ordering::SeqCst);
+    }
+}
+
+impl RegionView {
+    pub fn id(&self) -> &RegionId {
+        &self.id
     }
 
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
+    pub fn offset(&self) -> &u32 {
+        &self.offset
     }
-}
 
-impl<A> AsRef<[u8]> for ReadSlice<A>
-where
-    A: BufferAllocator,
-{
-    fn as_ref(&self) -> &[u8] {
-        self.buf.as_ref()
+    pub fn len(&self) -> &u32 {
+        &self.len
     }
-}
 
-impl<A> Drop for ReadSlice<A>
-where
-    A: BufferAllocator,
-{
-    fn drop(&mut self) {
-        if let Some(f) = self.cleanup.take() {
-            f();
-        }
+    pub fn key_len(&self) -> &u32 {
+        &self.key_len
+    }
+
+    pub fn value_len(&self) -> &u32 {
+        &self.value_len
+    }
+
+    pub fn refs(&self) -> &Arc<AtomicUsize> {
+        &self.refs
     }
 }
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index 2b1025ad..1d7ba647 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -118,7 +118,7 @@ where
     ///
     /// When all views from an allocation are dropped, the buffer will be released.
     #[tracing::instrument(skip(self))]
-    pub async fn allocate(self: &Arc<Self>, len: usize, sequence: Sequence) -> ViewMut {
+    pub async fn allocate(self: &Arc<Self>, len: usize, sequence: Sequence) -> RingBufferViewMut {
         loop {
             if let Some(view) = self.allocate_inner(len, sequence).await {
                 return view;
@@ -127,7 +127,11 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    async fn allocate_inner(self: &Arc<Self>, len: usize, sequence: Sequence) -> Option<ViewMut> {
+    async fn allocate_inner(
+        self: &Arc<Self>,
+        len: usize,
+        sequence: Sequence,
+    ) -> Option<RingBufferViewMut> {
         let len = align_up(self.align, len);
         let offset = self.allocated.fetch_add(len, Ordering::SeqCst);
 
@@ -158,7 +162,7 @@ where
 
         let ring = Arc::clone(self);
         let ring: Arc<dyn Ring> = ring;
-        Some(ViewMut::new(ring, offset, len, refs))
+        Some(RingBufferViewMut::new(ring, offset, len, refs))
     }
 
     fn refs(&self, sequence: Sequence) -> &Arc<AtomicUsize> {
@@ -216,7 +220,7 @@ where
 ///
 /// The underlying buffer of [`ViewMut`] must be valid during its lifetime.
 #[derive(Debug)]
-pub struct ViewMut {
+pub struct RingBufferViewMut {
     ring: Arc<dyn Ring>,
     ptr: *mut u8,
     offset: usize,
@@ -225,7 +229,7 @@ pub struct ViewMut {
     refs: Arc<AtomicUsize>,
 }
 
-impl ViewMut {
+impl RingBufferViewMut {
     fn new(ring: Arc<dyn Ring>, offset: usize, len: usize, refs: &Arc<AtomicUsize>) -> Self {
         refs.fetch_add(1, Ordering::AcqRel);
         let ptr = ring.ptr(offset);
@@ -251,12 +255,12 @@ impl ViewMut {
         align_up(self.ring.align(), self.len)
     }
 
-    pub fn freeze(self) -> View {
-        View::from(self)
+    pub fn freeze(self) -> RingBufferView {
+        RingBufferView::from(self)
     }
 }
 
-impl Deref for ViewMut {
+impl Deref for RingBufferViewMut {
     type Target = [u8];
 
     fn deref(&self) -> &Self::Target {
@@ -264,13 +268,13 @@ impl Deref for ViewMut {
     }
 }
 
-impl DerefMut for ViewMut {
+impl DerefMut for RingBufferViewMut {
     fn deref_mut(&mut self) -> &mut Self::Target {
         unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) }
     }
 }
 
-impl Drop for ViewMut {
+impl Drop for RingBufferViewMut {
     fn drop(&mut self) {
         if self.refs.fetch_sub(1, Ordering::AcqRel) == 1 {
             self.ring.release(self.offset..self.offset + self.capacity);
@@ -278,24 +282,24 @@ impl Drop for ViewMut {
     }
 }
 
-unsafe impl Send for ViewMut {}
-unsafe impl Sync for ViewMut {}
+unsafe impl Send for RingBufferViewMut {}
+unsafe impl Sync for RingBufferViewMut {}
 
 /// # Safety
 ///
 /// The underlying buffer of [`View`] must be valid during its lifetime.
 #[derive(Debug)]
-pub struct View {
-    view: ViewMut,
+pub struct RingBufferView {
+    view: RingBufferViewMut,
 }
 
-impl From<ViewMut> for View {
-    fn from(view: ViewMut) -> Self {
+impl From<RingBufferViewMut> for RingBufferView {
+    fn from(view: RingBufferViewMut) -> Self {
         Self { view }
     }
 }
 
-impl Deref for View {
+impl Deref for RingBufferView {
     type Target = [u8];
 
     fn deref(&self) -> &Self::Target {
@@ -303,10 +307,10 @@ impl Deref for View {
     }
 }
 
-impl Clone for View {
+impl Clone for RingBufferView {
     fn clone(&self) -> Self {
         self.view.refs.fetch_add(1, Ordering::AcqRel);
-        let view = ViewMut {
+        let view = RingBufferViewMut {
             ring: self.view.ring.clone(),
             ptr: self.view.ptr,
             offset: self.view.offset,
@@ -318,14 +322,14 @@ impl Clone for View {
     }
 }
 
-impl View {
+impl RingBufferView {
     pub fn aligned(&self) -> usize {
         self.view.aligned()
     }
 }
 
-unsafe impl Send for View {}
-unsafe impl Sync for View {}
+unsafe impl Send for RingBufferView {}
+unsafe impl Sync for RingBufferView {}
 
 #[cfg(test)]
 mod tests {

From cc87aaca820a452a5ca12e3bfa1f361afe5054ac Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 31 Oct 2023 14:20:28 +0800
Subject: [PATCH 160/261] refactor: remove slice (#196)

`Slice` and `SliceMut` are introduce to solve async io lifetime
problem in unsafe way. Remove slices to make code easier.

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/range.rs          |  4 +-
 foyer-storage-bench/src/analyze.rs | 45 ++++++++++++++
 foyer-storage/src/buffer.rs        |  5 +-
 foyer-storage/src/device/fs.rs     | 15 ++---
 foyer-storage/src/device/mod.rs    | 47 +++++++++++++--
 foyer-storage/src/lib.rs           |  1 -
 foyer-storage/src/region.rs        | 84 ++++++--------------------
 foyer-storage/src/slice.rs         | 97 ------------------------------
 8 files changed, 117 insertions(+), 181 deletions(-)
 delete mode 100644 foyer-storage/src/slice.rs

diff --git a/foyer-common/src/range.rs b/foyer-common/src/range.rs
index b01e35f7..14679269 100644
--- a/foyer-common/src/range.rs
+++ b/foyer-common/src/range.rs
@@ -89,14 +89,14 @@ pub trait RangeBoundsExt<T: Idx>: RangeBounds<T> {
         start..end
     }
 
-    fn len(&self) -> Option<T> {
+    fn size(&self) -> Option<T> {
         let start = self.start()?;
         let end = self.end()?;
         Some(end - start)
     }
 
     fn is_empty(&self) -> bool {
-        match self.len() {
+        match self.size() {
             Some(len) => len == ZeroOne::zero(),
             None => false,
         }
diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index 211f9046..d4d09d52 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -57,6 +57,9 @@ pub struct Analysis {
     insert_lat_p50: u64,
     insert_lat_p90: u64,
     insert_lat_p99: u64,
+    insert_lat_p999: u64,
+    insert_lat_p9999: u64,
+    insert_lat_p99999: u64,
     insert_lat_pmax: u64,
 
     get_iops: f64,
@@ -65,10 +68,16 @@ pub struct Analysis {
     get_miss_lat_p50: u64,
     get_miss_lat_p90: u64,
     get_miss_lat_p99: u64,
+    get_miss_lat_p999: u64,
+    get_miss_lat_p9999: u64,
+    get_miss_lat_p99999: u64,
     get_miss_lat_pmax: u64,
     get_hit_lat_p50: u64,
     get_hit_lat_p90: u64,
     get_hit_lat_p99: u64,
+    get_hit_lat_p999: u64,
+    get_hit_lat_p9999: u64,
+    get_hit_lat_p99999: u64,
     get_hit_lat_pmax: u64,
 }
 
@@ -79,6 +88,9 @@ pub struct MetricsDump {
     pub insert_lat_p50: u64,
     pub insert_lat_p90: u64,
     pub insert_lat_p99: u64,
+    pub insert_lat_p999: u64,
+    pub insert_lat_p9999: u64,
+    pub insert_lat_p99999: u64,
     pub insert_lat_pmax: u64,
 
     pub get_ios: usize,
@@ -87,10 +99,16 @@ pub struct MetricsDump {
     pub get_hit_lat_p50: u64,
     pub get_hit_lat_p90: u64,
     pub get_hit_lat_p99: u64,
+    pub get_hit_lat_p999: u64,
+    pub get_hit_lat_p9999: u64,
+    pub get_hit_lat_p99999: u64,
     pub get_hit_lat_pmax: u64,
     pub get_miss_lat_p50: u64,
     pub get_miss_lat_p90: u64,
     pub get_miss_lat_p99: u64,
+    pub get_miss_lat_p999: u64,
+    pub get_miss_lat_p9999: u64,
+    pub get_miss_lat_p99999: u64,
     pub get_miss_lat_pmax: u64,
 }
 
@@ -141,6 +159,9 @@ impl Metrics {
             insert_lat_p50: insert_lats.value_at_quantile(0.5),
             insert_lat_p90: insert_lats.value_at_quantile(0.9),
             insert_lat_p99: insert_lats.value_at_quantile(0.99),
+            insert_lat_p999: insert_lats.value_at_quantile(0.999),
+            insert_lat_p9999: insert_lats.value_at_quantile(0.9999),
+            insert_lat_p99999: insert_lats.value_at_quantile(0.99999),
             insert_lat_pmax: insert_lats.max(),
 
             get_ios: self.get_ios.load(Ordering::Relaxed),
@@ -149,10 +170,16 @@ impl Metrics {
             get_hit_lat_p50: get_hit_lats.value_at_quantile(0.5),
             get_hit_lat_p90: get_hit_lats.value_at_quantile(0.9),
             get_hit_lat_p99: get_hit_lats.value_at_quantile(0.99),
+            get_hit_lat_p999: get_hit_lats.value_at_quantile(0.999),
+            get_hit_lat_p9999: get_hit_lats.value_at_quantile(0.9999),
+            get_hit_lat_p99999: get_hit_lats.value_at_quantile(0.99999),
             get_hit_lat_pmax: get_hit_lats.max(),
             get_miss_lat_p50: get_miss_lats.value_at_quantile(0.5),
             get_miss_lat_p90: get_miss_lats.value_at_quantile(0.9),
             get_miss_lat_p99: get_miss_lats.value_at_quantile(0.99),
+            get_miss_lat_p999: get_miss_lats.value_at_quantile(0.999),
+            get_miss_lat_p9999: get_miss_lats.value_at_quantile(0.9999),
+            get_miss_lat_p99999: get_miss_lats.value_at_quantile(0.99999),
             get_miss_lat_pmax: get_miss_lats.max(),
         }
     }
@@ -199,6 +226,9 @@ impl std::fmt::Display for Analysis {
         writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
         writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
         writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
+        writeln!(f, "insert lat p999: {}us", self.insert_lat_p999)?;
+        writeln!(f, "insert lat p9999: {}us", self.insert_lat_p9999)?;
+        writeln!(f, "insert lat p99999: {}us", self.insert_lat_p99999)?;
         writeln!(f, "insert lat pmax: {}us", self.insert_lat_pmax)?;
 
         // get statics
@@ -209,10 +239,16 @@ impl std::fmt::Display for Analysis {
         writeln!(f, "get hit lat p50: {}us", self.get_hit_lat_p50)?;
         writeln!(f, "get hit lat p90: {}us", self.get_hit_lat_p90)?;
         writeln!(f, "get hit lat p99: {}us", self.get_hit_lat_p99)?;
+        writeln!(f, "get hit lat p999: {}us", self.get_hit_lat_p999)?;
+        writeln!(f, "get hit lat p9999: {}us", self.get_hit_lat_p9999)?;
+        writeln!(f, "get hit lat p99999: {}us", self.get_hit_lat_p99999)?;
         writeln!(f, "get hit lat pmax: {}us", self.get_hit_lat_pmax)?;
         writeln!(f, "get miss lat p50: {}us", self.get_miss_lat_p50)?;
         writeln!(f, "get miss lat p90: {}us", self.get_miss_lat_p90)?;
         writeln!(f, "get miss lat p99: {}us", self.get_miss_lat_p99)?;
+        writeln!(f, "get miss lat p999: {}us", self.get_miss_lat_p999)?;
+        writeln!(f, "get miss lat p9999: {}us", self.get_miss_lat_p9999)?;
+        writeln!(f, "get miss lat p99999: {}us", self.get_miss_lat_p99999)?;
         writeln!(f, "get miss lat pmax: {}us", self.get_miss_lat_pmax)?;
 
         Ok(())
@@ -254,6 +290,9 @@ pub fn analyze(
         insert_lat_p50: metrics_dump_end.insert_lat_p50,
         insert_lat_p90: metrics_dump_end.insert_lat_p90,
         insert_lat_p99: metrics_dump_end.insert_lat_p99,
+        insert_lat_p999: metrics_dump_end.insert_lat_p999,
+        insert_lat_p9999: metrics_dump_end.insert_lat_p9999,
+        insert_lat_p99999: metrics_dump_end.insert_lat_p99999,
         insert_lat_pmax: metrics_dump_end.insert_lat_pmax,
 
         get_iops,
@@ -262,10 +301,16 @@ pub fn analyze(
         get_hit_lat_p50: metrics_dump_end.get_hit_lat_p50,
         get_hit_lat_p90: metrics_dump_end.get_hit_lat_p90,
         get_hit_lat_p99: metrics_dump_end.get_hit_lat_p99,
+        get_hit_lat_p999: metrics_dump_end.get_hit_lat_p999,
+        get_hit_lat_p9999: metrics_dump_end.get_hit_lat_p9999,
+        get_hit_lat_p99999: metrics_dump_end.get_hit_lat_p99999,
         get_hit_lat_pmax: metrics_dump_end.get_hit_lat_pmax,
         get_miss_lat_p50: metrics_dump_end.get_miss_lat_p50,
         get_miss_lat_p90: metrics_dump_end.get_miss_lat_p90,
         get_miss_lat_p99: metrics_dump_end.get_miss_lat_p99,
+        get_miss_lat_p999: metrics_dump_end.get_miss_lat_p999,
+        get_miss_lat_p9999: metrics_dump_end.get_miss_lat_p9999,
+        get_miss_lat_p99999: metrics_dump_end.get_miss_lat_p99999,
         get_miss_lat_pmax: metrics_dump_end.get_miss_lat_pmax,
     }
 }
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index aaec2741..68b9b68d 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -127,10 +127,7 @@ where
         debug_assert!(self.offset + buffer.len() <= self.device.region_size());
 
         // flush and clear buffer
-        let (res, mut buffer) = self
-            .device
-            .write(buffer, .., region, self.offset as u64)
-            .await;
+        let (res, mut buffer) = self.device.write(buffer, .., region, self.offset).await;
         buffer.clear();
         self.buffer = Some(buffer);
         res?;
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index e042c08a..a4301b35 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -87,7 +87,7 @@ impl Device for FsDevice {
         buf: B,
         range: impl IoRange,
         region: RegionId,
-        offset: u64,
+        offset: usize,
     ) -> (DeviceResult<usize>, B)
     where
         B: IoBuf,
@@ -95,10 +95,10 @@ impl Device for FsDevice {
         let file_capacity = self.inner.config.file_capacity;
 
         let range = range.bounds(0..buf.as_ref().len());
-        let len = RangeBoundsExt::len(&range).unwrap();
+        let len = RangeBoundsExt::size(&range).unwrap();
 
         assert!(
-            offset as usize + len <= file_capacity,
+            offset + len <= file_capacity,
             "offset ({offset}) + len ({len}) <= file capacity ({file_capacity})"
         );
 
@@ -118,7 +118,7 @@ impl Device for FsDevice {
         mut buf: B,
         range: impl IoRange,
         region: RegionId,
-        offset: u64,
+        offset: usize,
     ) -> (DeviceResult<usize>, B)
     where
         B: IoBufMut,
@@ -126,10 +126,10 @@ impl Device for FsDevice {
         let file_capacity = self.inner.config.file_capacity;
 
         let range = range.bounds(0..buf.as_ref().len());
-        let len = RangeBoundsExt::len(&range).unwrap();
+        let len = RangeBoundsExt::size(&range).unwrap();
 
         assert!(
-            offset as usize + len <= file_capacity,
+            offset + len <= file_capacity,
             "offset ({offset}) + len ({len}) <= file capacity ({file_capacity})"
         );
 
@@ -151,15 +151,12 @@ impl Device for FsDevice {
         //
         // See also [syncfs(2)](https://man7.org/linux/man-pages/man2/sync.2.html)
         asyncify(move || nix::unistd::syncfs(fd).map_err(DeviceError::from)).await?;
-
-        // TODO(MrCroxx): track dirty files and call fsync(2) on them on other target os.
         Ok(())
     }
 
     #[cfg(not(target_os = "linux"))]
     async fn flush(&self) -> DeviceResult<()> {
         // TODO(MrCroxx): track dirty files and call fsync(2) on them on other target os.
-
         Ok(())
     }
 
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 3780ae2e..45eddc75 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -41,7 +41,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
         buf: B,
         range: impl IoRange,
         region: RegionId,
-        offset: u64,
+        offset: usize,
     ) -> impl Future<Output = (DeviceResult<usize>, B)> + Send
     where
         B: IoBuf;
@@ -52,7 +52,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
         buf: B,
         range: impl IoRange,
         region: RegionId,
-        offset: u64,
+        offset: usize,
     ) -> impl Future<Output = (DeviceResult<usize>, B)> + Send
     where
         B: IoBufMut;
@@ -64,8 +64,10 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
 
     fn regions(&self) -> usize;
 
+    /// must be power of 2
     fn align(&self) -> usize;
 
+    /// optimized io size
     fn io_size(&self) -> usize;
 
     fn io_buffer_allocator(&self) -> &Self::IoBufferAllocator;
@@ -78,6 +80,43 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     }
 }
 
+pub trait DeviceExt: Device {
+    #[must_use]
+    fn load(
+        &self,
+        region: RegionId,
+        range: impl IoRange,
+    ) -> impl Future<Output = DeviceResult<Vec<u8, Self::IoBufferAllocator>>> + Send {
+        async move {
+            let range = range.bounds(0..self.region_size());
+            let size = range.size().unwrap();
+            debug_assert_eq!(size & (self.align() - 1), 0);
+
+            let mut buf = self.io_buffer(size, size);
+            let mut offset = 0;
+
+            while range.start + offset < range.end {
+                let len = std::cmp::min(self.io_size(), size - offset);
+                let (res, b) = self
+                    .read(buf, offset..offset + len, region, range.start + offset)
+                    .await;
+                let bytes = res?;
+                offset += bytes;
+                buf = b;
+                if bytes != len {
+                    break;
+                }
+            }
+
+            unsafe { buf.set_len(offset) };
+
+            Ok(buf)
+        }
+    }
+}
+
+impl<D: Device> DeviceExt for D {}
+
 #[cfg(not(madsim))]
 #[tracing::instrument(level = "trace", skip(f))]
 async fn asyncify<F, T>(f: F) -> T
@@ -124,7 +163,7 @@ pub mod tests {
             buf: B,
             _range: impl IoRange,
             _region: RegionId,
-            _offset: u64,
+            _offset: usize,
         ) -> (DeviceResult<usize>, B)
         where
             B: IoBuf,
@@ -137,7 +176,7 @@ pub mod tests {
             buf: B,
             _range: impl IoRange,
             _region: RegionId,
-            _offset: u64,
+            _offset: usize,
         ) -> (DeviceResult<usize>, B)
         where
             B: IoBufMut,
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index f46a43c5..24089840 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -38,7 +38,6 @@ pub mod region_manager;
 pub mod reinsertion;
 pub mod ring;
 pub mod runtime;
-pub mod slice;
 pub mod storage;
 pub mod store;
 
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 4320a8a7..46366cc2 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -13,7 +13,8 @@
 //  limitations under the License.
 
 use bytes::{Buf, BufMut};
-use parking_lot::{Mutex, MutexGuard};
+use foyer_common::range::RangeBoundsExt;
+use parking_lot::Mutex;
 use std::{
     collections::btree_map::{BTreeMap, Entry},
     fmt::Debug,
@@ -26,9 +27,8 @@ use std::{
 use tokio::sync::oneshot;
 
 use crate::{
-    device::{BufferAllocator, Device},
+    device::{BufferAllocator, Device, DeviceExt},
     error::Result,
-    slice::SliceMut,
 };
 
 pub type RegionId = u32;
@@ -75,14 +75,6 @@ where
     refs: Arc<AtomicUsize>,
 }
 
-/// [`Region`] represents a contiguous aligned range on device and its optional dirty buffer.
-///
-/// [`Region`] may be in one of the following states:
-///
-/// - physical write : written by flushers with append pattern
-/// - physical read  : read if entry is not in ring buffer
-/// - reclaim        : happens after the region is evicted, must guarantee there is no writers or readers,
-///                    *or in-flight writers or readers*
 impl<D> Region<D>
 where
     D: Device,
@@ -137,22 +129,13 @@ where
         &self,
         range: impl RangeBounds<usize>,
     ) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
-        let start = match range.start_bound() {
-            std::ops::Bound::Included(i) => *i,
-            std::ops::Bound::Excluded(i) => *i + 1,
-            std::ops::Bound::Unbounded => 0,
-        };
-        let end = match range.end_bound() {
-            std::ops::Bound::Included(i) => *i + 1,
-            std::ops::Bound::Excluded(i) => *i,
-            std::ops::Bound::Unbounded => self.device.region_size(),
-        };
+        let range = range.bounds(0..self.device.region_size());
 
         let rx = {
             let mut inner = self.inner.lock();
 
             // join wait map if exists
-            let rx = match inner.waits.entry((start, end)) {
+            let rx = match inner.waits.entry((range.start, range.end)) {
                 Entry::Vacant(v) => {
                     v.insert(vec![]);
                     None
@@ -176,45 +159,23 @@ where
 
         // otherwise, read from device
         let region = self.id;
-        let mut buf = self.device.io_buffer(end - start, end - start);
-
-        let mut offset = 0;
-        while start + offset < end {
-            let len = std::cmp::min(self.device.io_size(), end - start - offset);
-            tracing::trace!(
-                "read region {} [{}..{}]",
-                region,
-                start + offset,
-                start + offset + len
-            );
-            let s = unsafe { SliceMut::new(&mut buf[offset..offset + len]) };
-            let (res, _s) = self
-                .device
-                .read(s, .., region, (start + offset) as u64)
-                .await;
-            let read = match res {
-                Ok(bytes) => bytes,
-                Err(e) => {
-                    let mut inner = self.inner.lock();
-                    self.cleanup(&mut inner, start, end)?;
-                    return Err(e.into());
-                }
-            };
-            if read != len {
-                let mut inner = self.inner.lock();
-                self.cleanup(&mut inner, start, end)?;
-                // TODO(MrCroxx): return err?
+
+        let buf = match self.device.load(region, range.start..range.end).await {
+            Err(e) => {
+                self.cleanup(range.start, range.end)?;
+                return Err(e.into());
+            }
+            Ok(buf) if buf.len() != range.size().unwrap() => {
+                self.cleanup(range.start, range.end)?;
                 return Ok(None);
             }
-            offset += len;
-        }
+            Ok(buf) => buf,
+        };
         let buf = Arc::new(buf);
 
-        if let Some(txs) = self.inner.lock().waits.remove(&(start, end)) {
-            // TODO: handle error !!!!!!!!!!!
+        if let Some(txs) = self.inner.lock().waits.remove(&(range.start, range.end)) {
             for tx in txs {
-                tx.send(Ok(buf.clone()))
-                    .map_err(|_| anyhow::anyhow!("fail to send load result"))?;
+                tx.send(Ok(buf.clone())).unwrap()
             }
         }
 
@@ -230,16 +191,11 @@ where
     }
 
     /// Cleanup waits.
-    fn cleanup(
-        &self,
-        guard: &mut MutexGuard<'_, RegionInner<D::IoBufferAllocator>>,
-        start: usize,
-        end: usize,
-    ) -> Result<()> {
-        if let Some(txs) = guard.waits.remove(&(start, end)) {
+    fn cleanup(&self, start: usize, end: usize) -> Result<()> {
+        if let Some(txs) = self.inner.lock().waits.remove(&(start, end)) {
             for tx in txs {
                 tx.send(Err(anyhow::anyhow!("cancelled by previous error").into()))
-                    .map_err(|_| anyhow::anyhow!("fail to cleanup waits"))?;
+                    .unwrap()
             }
         }
         Ok(())
diff --git a/foyer-storage/src/slice.rs b/foyer-storage/src/slice.rs
deleted file mode 100644
index 7eb7d1df..00000000
--- a/foyer-storage/src/slice.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-/// A mutable bytes slice without lifetime checks.
-///
-/// `slice` does NOT care about buffer allocator, because it never allocates or deallocatoes any buffer.
-#[derive(Debug)]
-pub struct SliceMut {
-    ptr: *mut u8,
-    len: usize,
-}
-
-unsafe impl Send for SliceMut {}
-unsafe impl Sync for SliceMut {}
-
-impl SliceMut {
-    /// # Safety
-    ///
-    /// `buf` MUST outlive `SliceMut`.
-    pub unsafe fn new(buf: &mut [u8]) -> Self {
-        let ptr = buf.as_mut_ptr();
-        let len = buf.len();
-        Self { ptr, len }
-    }
-
-    pub fn len(&self) -> usize {
-        self.len
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    pub fn seal(self) -> Slice {
-        unsafe { Slice::new(self.as_ref()) }
-    }
-}
-
-impl AsRef<[u8]> for SliceMut {
-    fn as_ref(&self) -> &[u8] {
-        unsafe { core::slice::from_raw_parts(self.ptr, self.len) }
-    }
-}
-
-impl AsMut<[u8]> for SliceMut {
-    fn as_mut(&mut self) -> &mut [u8] {
-        unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) }
-    }
-}
-
-/// An immutable bytes slice without lifetime checks.
-///
-/// `slice` does NOT care about buffer allocator, because it never allocates or deallocatoes any buffer.
-#[derive(Debug)]
-pub struct Slice {
-    ptr: *const u8,
-    len: usize,
-}
-
-unsafe impl Send for Slice {}
-unsafe impl Sync for Slice {}
-
-impl Slice {
-    /// # Safety
-    ///
-    /// `buf` MUST outlive `Slice`.
-    pub unsafe fn new(buf: &[u8]) -> Self {
-        let ptr = buf.as_ptr();
-        let len = buf.len();
-        Self { ptr, len }
-    }
-
-    pub fn len(&self) -> usize {
-        self.len
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-}
-
-impl AsRef<[u8]> for Slice {
-    fn as_ref(&self) -> &[u8] {
-        unsafe { core::slice::from_raw_parts(self.ptr, self.len) }
-    }
-}

From 929c06adc653e48195807ed61d95e16a0198308d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 1 Nov 2023 00:46:21 +0800
Subject: [PATCH 161/261] refactor: refine flush buffer, at most write once per
 entry (#197)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs     |   4 +
 foyer-storage/src/buffer.rs         | 134 +++++++++++++---------------
 foyer-storage/src/flusher.rs        |   3 +-
 foyer-storage/src/generic.rs        |   8 ++
 foyer-storage/src/lazy.rs           |   2 +
 foyer-storage/src/region.rs         |   4 -
 foyer-storage/tests/storage_test.rs |   4 +
 7 files changed, 82 insertions(+), 77 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 94e57396..85dc087a 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -111,6 +111,9 @@ pub struct Args {
     #[arg(long, default_value_t = 1024)]
     buffer_pool_size: usize,
 
+    #[arg(long, default_value_t = 0)]
+    flusher_buffer_size: usize,
+
     /// (MiB)
     #[arg(long, default_value_t = 1024)]
     ring_buffer_capacity: usize,
@@ -561,6 +564,7 @@ async fn main() {
         admissions,
         reinsertions,
         buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
+        flusher_buffer_size: args.flusher_buffer_size,
         flushers: args.flushers,
         flush_rate_limit: args.flush_rate_limit * 1024 * 1024,
         reclaimers: args.reclaimers,
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 68b9b68d..789875ce 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -42,12 +42,9 @@ pub struct FlushBuffer<D>
 where
     D: Device,
 {
+    // TODO(MrCroxx): optimize buffer allocation
     /// io buffer
-    ///
-    /// # Safety
-    ///
-    /// `buffer` should always be `Some`. The usage of `Option` is for temporarily taking ownership.
-    buffer: Option<Vec<u8, D::IoBufferAllocator>>,
+    buffer: Vec<u8, D::IoBufferAllocator>,
 
     /// current writing region
     region: Option<RegionId>,
@@ -60,20 +57,24 @@ where
 
     // underlying device
     device: D,
+
+    default_buffer_capacity: usize,
 }
 
 impl<D> FlushBuffer<D>
 where
     D: Device,
 {
-    pub fn new(device: D) -> Self {
-        let buffer = Some(device.io_buffer(0, device.io_size()));
+    pub fn new(device: D, default_buffer_capacity: usize) -> Self {
+        let default_buffer_capacity = std::cmp::max(default_buffer_capacity, device.io_size());
+        let buffer = device.io_buffer(0, default_buffer_capacity);
         Self {
             buffer,
             region: None,
             offset: 0,
             entries: vec![],
             device,
+            default_buffer_capacity,
         }
     }
 
@@ -83,9 +84,12 @@ where
 
     pub fn remaining(&self) -> usize {
         if self.region.is_none() {
-            return 0;
+            0
+        } else {
+            self.device
+                .region_size()
+                .saturating_sub(self.offset + self.buffer.len())
         }
-        self.device.region_size() - self.offset - self.buffer.as_ref().unwrap().len()
     }
 
     /// Flush io buffer if necessary, and reset io buffer to a new region.
@@ -93,17 +97,17 @@ where
     /// Returns fully flushed entries.
     pub async fn rotate(&mut self, region: RegionId) -> BufferResult<Vec<PositionedEntry>> {
         let entries = self.flush().await?;
-        debug_assert!(self.buffer.as_ref().unwrap().is_empty());
+        debug_assert!(self.buffer.is_empty());
         self.region = Some(region);
         self.offset = 0;
 
         // write region header
-        let buffer = self.buffer.as_mut().unwrap();
-        unsafe { buffer.set_len(self.device.align()) };
+        unsafe { self.buffer.set_len(self.device.align()) };
         let header = RegionHeader {
             magic: REGION_MAGIC,
         };
-        header.write(buffer);
+        header.write(&mut self.buffer[..]);
+        debug_assert_eq!(self.buffer.len(), self.device.align());
 
         Ok(entries)
     }
@@ -120,20 +124,20 @@ where
         };
 
         // align io buffer
-        let mut buffer = self.buffer.take().unwrap();
-        let len = align_up(self.device.align(), buffer.len());
-        debug_assert!(len <= buffer.capacity());
-        unsafe { buffer.set_len(len) };
-        debug_assert!(self.offset + buffer.len() <= self.device.region_size());
+        let len = align_up(self.device.align(), self.buffer.len());
+        debug_assert!(len <= self.buffer.capacity());
+        unsafe { self.buffer.set_len(len) };
+        debug_assert!(self.offset + self.buffer.len() <= self.device.region_size());
 
         // flush and clear buffer
-        let (res, mut buffer) = self.device.write(buffer, .., region, self.offset).await;
-        buffer.clear();
-        self.buffer = Some(buffer);
+        let mut buf = self.device.io_buffer(0, self.default_buffer_capacity);
+        std::mem::swap(&mut self.buffer, &mut buf);
+
+        let (res, _buf) = self.device.write(buf, .., region, self.offset).await;
         res?;
 
         // advance io buffer
-        self.offset += self.device.io_size();
+        self.offset += len;
         if self.offset == self.device.region_size() {
             self.region = None;
         }
@@ -145,62 +149,38 @@ where
 
     /// Write entry to io buffer.
     ///
-    /// The io buffer may be flushed if needed.
+    /// The io buffer may be flushed if buffer size equals or exceeds device io size.
     ///
     /// Returns fully flushed entries if there is enough space in the current region.
     /// Otherwise, returns `NotEnough` error with the given `entry`.
     pub async fn write(&mut self, entry: Entry) -> BufferResult<Vec<PositionedEntry>> {
         // check region remaining size
-        let padding = align_up(self.device.align(), entry.view.len()) - entry.view.len();
-        if self.remaining() < entry.view.len() + padding {
+        let len = align_up(self.device.align(), entry.view.len());
+        if self.remaining() < len {
             return Err(BufferError::NotEnough { entry });
         }
 
+        // write view and padding
         let region = self.region.unwrap();
-        let offset = self.offset + self.buffer.as_ref().unwrap().len();
-
-        let mut entries = vec![];
-        let mut written = 0;
-
-        // write view
-        let mut flushed = true;
-        while written < entry.view.len() {
-            flushed = false;
-            let buffer = self.buffer.as_mut().unwrap();
-            let bytes = std::cmp::min(
-                entry.view.len() - written,
-                self.device.io_size() - buffer.len(),
-            );
-            std::io::copy(&mut &entry.view[written..written + bytes], buffer)
-                .map_err(DeviceError::from)?;
-            written += bytes;
-
-            if buffer.len() == self.device.io_size() {
-                entries.append(&mut self.flush().await?);
-                flushed = true;
-            }
-        }
-
-        // write padding
-        let buffer = self.buffer.as_mut().unwrap();
-        debug_assert!(self.device.io_size() - buffer.len() >= padding);
-        unsafe { buffer.set_len(buffer.len() + padding) };
-        debug_assert!(is_aligned(self.device.align(), buffer.len()));
-        if buffer.len() == self.device.io_size() {
-            entries.append(&mut self.flush().await?);
-            flushed = true;
-        }
-
-        let entry = PositionedEntry {
+        let offset = self.offset + self.buffer.len();
+        let target_len = self.buffer.len() + len;
+        debug_assert!(is_aligned(self.device.align(), offset));
+
+        self.buffer.reserve_exact(len);
+        std::io::copy(&mut &entry.view[..], &mut self.buffer).map_err(DeviceError::from)?;
+        unsafe { self.buffer.set_len(target_len) };
+        self.entries.push(PositionedEntry {
             entry,
             region,
             offset,
-        };
-        if flushed {
-            entries.push(entry);
+        });
+
+        // flush if buffer equals or exceeds device io size
+        let entries = if self.buffer.len() >= self.device.io_size() || self.remaining() == 0 {
+            self.flush().await?
         } else {
-            self.entries.push(entry);
-        }
+            vec![]
+        };
 
         Ok(entries)
     }
@@ -244,8 +224,9 @@ mod tests {
         })
         .await
         .unwrap();
+        const DEFAULT_BUFFER_CAPACITY: usize = 32 * 1024;
 
-        let mut buffer = FlushBuffer::new(device.clone());
+        let mut buffer = FlushBuffer::new(device.clone(), DEFAULT_BUFFER_CAPACITY);
         assert_eq!(buffer.region(), None);
 
         {
@@ -267,21 +248,28 @@ mod tests {
             let entries = buffer.rotate(0).await.unwrap();
             assert!(entries.is_empty());
 
+            // 4 ~ 12 KiB
             let entries = buffer.write(entry.clone()).await.unwrap();
             assert!(entries.is_empty());
+            // 12 ~ 20 KiB
             let entries = buffer.write(entry.clone()).await.unwrap();
-            assert_eq!(entries.len(), 1);
+            assert_eq!(entries.len(), 2);
             assert_eq!(entries[0].offset, 4 * 1024);
+            assert_eq!(entries[1].offset, 12 * 1024);
 
+            // 20 ~ 28 KiB
+            let entries = buffer.write(entry.clone()).await.unwrap();
+            assert!(entries.is_empty());
             let entries = buffer.flush().await.unwrap();
             assert_eq!(entries.len(), 1);
-            assert_eq!(entries[0].offset, 12 * 1024);
+            assert_eq!(entries[0].offset, 20 * 1024);
 
             let buf = device.io_buffer(64 * 1024, 64 * 1024);
             let (res, buf) = device.read(buf, .., 0, 0).await;
             res.unwrap();
             assert_eq!(&buf[4 * 1024..9 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
             assert_eq!(&buf[12 * 1024..17 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
+            assert_eq!(&buf[20 * 1024..25 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
 
             assert!(buffer.entries.is_empty());
         }
@@ -307,8 +295,10 @@ mod tests {
             let entries = buffer.rotate(1).await.unwrap();
             assert!(entries.is_empty());
 
+            // 4 ~ 60 KiB
             let entries = buffer.write(entry).await.unwrap();
-            assert!(entries.is_empty());
+            assert_eq!(entries.len(), 1);
+            assert_eq!(entries[0].offset, 4 * 1024);
 
             let view = {
                 let mut view = ring.allocate(3 * 1024 - 128, 2).await; // ~ 3 KiB
@@ -317,10 +307,10 @@ mod tests {
             };
             let entry = ent(view);
 
+            // 60 ~ 64 KiB
             let entries = buffer.write(entry).await.unwrap();
-            assert_eq!(entries.len(), 2);
-            assert_eq!(entries[0].offset, 4 * 1024);
-            assert_eq!(entries[1].offset, 60 * 1024);
+            assert_eq!(entries.len(), 1);
+            assert_eq!(entries[0].offset, 60 * 1024);
 
             let buf = device.io_buffer(64 * 1024, 64 * 1024);
             let (res, buf) = device.read(buf, .., 1, 0).await;
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index f404d3ee..3df3c86a 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -94,6 +94,7 @@ where
     EL: Link,
 {
     pub fn new(
+        default_buffer_capacity: usize,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         catalog: Arc<Catalog<K>>,
         device: D,
@@ -101,7 +102,7 @@ where
         metrics: Arc<Metrics>,
         stop_rx: broadcast::Receiver<()>,
     ) -> Self {
-        let buffer = FlushBuffer::new(device.clone());
+        let buffer = FlushBuffer::new(device.clone(), default_buffer_capacity);
         Self {
             region_manager,
             catalog,
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 1ef39808..09d51299 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -95,6 +95,9 @@ where
     /// Buffer pool size, should be a multiplier of device region size.
     pub buffer_pool_size: usize,
 
+    /// Flusher default buffer capacity, must be equals or larger than device io size.
+    pub flusher_buffer_size: usize,
+
     /// Count of flushers.
     pub flushers: usize,
 
@@ -136,6 +139,7 @@ where
             .field("admissions", &self.admissions)
             .field("reinsertions", &self.reinsertions)
             .field("buffer_pool_size", &self.buffer_pool_size)
+            .field("flusher_buffer_size", &self.flusher_buffer_size)
             .field("flushers", &self.flushers)
             .field("flush_rate_limit", &self.flush_rate_limit)
             .field("reclaimers", &self.reclaimers)
@@ -165,6 +169,7 @@ where
             admissions: self.admissions.clone(),
             reinsertions: self.reinsertions.clone(),
             buffer_pool_size: self.buffer_pool_size,
+            flusher_buffer_size: self.flusher_buffer_size,
             flushers: self.flushers,
             flush_rate_limit: self.flush_rate_limit,
             reclaimers: self.reclaimers,
@@ -333,6 +338,7 @@ where
             .zip_eq(flusher_entry_rxs.into_iter())
             .map(|(stop_rx, entry_rx)| {
                 Flusher::new(
+                    config.flusher_buffer_size,
                     region_manager.clone(),
                     catalog.clone(),
                     device.clone(),
@@ -1191,6 +1197,7 @@ mod tests {
             admissions,
             reinsertions,
             buffer_pool_size: 8 * MB,
+            flusher_buffer_size: 0,
             flushers: 1,
             flush_rate_limit: 0,
             reclaimers: 1,
@@ -1245,6 +1252,7 @@ mod tests {
             admissions: vec![],
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
+            flusher_buffer_size: 0,
             flushers: 1,
             flush_rate_limit: 0,
             reclaimers: 0,
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 2214f669..3bad82f0 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -244,6 +244,7 @@ mod tests {
             admissions: vec![],
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
+            flusher_buffer_size: 0,
             flushers: 1,
             flush_rate_limit: 0,
             reclaimers: 1,
@@ -281,6 +282,7 @@ mod tests {
             admissions: vec![],
             reinsertions: vec![],
             buffer_pool_size: 8 * MB,
+            flusher_buffer_size: 0,
             flushers: 1,
             flush_rate_limit: 0,
             reclaimers: 1,
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 46366cc2..1ad06e7c 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -202,10 +202,6 @@ where
     }
 }
 
-// read & write slice
-
-pub trait CleanupFn = FnOnce() + Send + Sync + 'static;
-
 #[derive(Debug)]
 pub struct RegionView {
     id: RegionId,
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 58977bb7..0b707148 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -134,6 +134,7 @@ async fn test_store() {
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
         buffer_pool_size: 2 * MB,
+        flusher_buffer_size: 0,
         flushers: 1,
         flush_rate_limit: 0,
         reclaimers: 1,
@@ -166,6 +167,7 @@ async fn test_lazy_store() {
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
         buffer_pool_size: 2 * MB,
+        flusher_buffer_size: 0,
         flushers: 1,
         flush_rate_limit: 0,
         reclaimers: 1,
@@ -199,6 +201,7 @@ async fn test_runtime_store() {
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
             buffer_pool_size: 2 * MB,
+            flusher_buffer_size: 0,
             flushers: 1,
             flush_rate_limit: 0,
             reclaimers: 1,
@@ -238,6 +241,7 @@ async fn test_runtime_lazy_store() {
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
             buffer_pool_size: 2 * MB,
+            flusher_buffer_size: 0,
             flushers: 1,
             flush_rate_limit: 0,
             reclaimers: 1,

From 3578fb49acb3a27162a7f68a47d11e2658ba00e3 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 1 Nov 2023 16:50:36 +0800
Subject: [PATCH 162/261] feat: support zstd compression (#198)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/code.rs            |  70 ++++++---
 foyer-storage-bench/src/main.rs     |  12 ++
 foyer-storage/Cargo.toml            |   1 +
 foyer-storage/src/buffer.rs         | 231 ++++++++++++++++++++++++----
 foyer-storage/src/compress.rs       |  38 +++++
 foyer-storage/src/flusher.rs        |  22 ++-
 foyer-storage/src/generic.rs        | 143 ++++++++---------
 foyer-storage/src/lazy.rs           |   2 +
 foyer-storage/src/lib.rs            |   1 +
 foyer-storage/tests/storage_test.rs |  39 +++++
 foyer-workspace-hack/Cargo.toml     |   1 +
 11 files changed, 428 insertions(+), 132 deletions(-)
 create mode 100644 foyer-storage/src/compress.rs

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 275a315f..777dae8f 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -29,22 +29,18 @@ pub trait Key:
     + Clone
     + std::fmt::Debug
 {
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Key` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
@@ -52,22 +48,18 @@ pub trait Key:
 
 #[expect(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn serialized_len(&self) -> usize {
         panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn write(&self, buf: &mut [u8]) {
         panic!("Method `write` must be implemented for `Value` if storage is used.")
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn read(buf: &[u8]) -> Self {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
@@ -87,17 +79,17 @@ macro_rules! impl_key {
         paste! {
             $(
                 impl Key for $type {
-                    #[cfg_attr(coverage_nightly, coverage(off))]
+
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
                     }
 
-                    #[cfg_attr(coverage_nightly, coverage(off))]
+
                     fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
-                    #[cfg_attr(coverage_nightly, coverage(off))]
+
                     fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
@@ -112,17 +104,17 @@ macro_rules! impl_value {
         paste! {
             $(
                 impl Value for $type {
-                    #[cfg_attr(coverage_nightly, coverage(off))]
+
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
                     }
 
-                    #[cfg_attr(coverage_nightly, coverage(off))]
+
                     fn write(&self, mut buf: &mut [u8]) {
                         buf.[< put_ $type>](*self)
                     }
 
-                    #[cfg_attr(coverage_nightly, coverage(off))]
+
                     fn read(mut buf: &[u8]) -> Self {
                         buf.[< get_ $type>]()
                     }
@@ -135,24 +127,66 @@ macro_rules! impl_value {
 for_all_primitives! { impl_key }
 for_all_primitives! { impl_value }
 
+impl Key for Vec<u8> {
+    fn weight(&self) -> usize {
+        self.len()
+    }
+
+    fn serialized_len(&self) -> usize {
+        self.len()
+    }
+
+    fn write(&self, mut buf: &mut [u8]) {
+        buf.put_slice(self);
+    }
+
+    fn read(buf: &[u8]) -> Self {
+        buf.to_vec()
+    }
+}
+
 impl Value for Vec<u8> {
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn weight(&self) -> usize {
         self.len()
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn serialized_len(&self) -> usize {
         self.len()
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn write(&self, mut buf: &mut [u8]) {
         buf.put_slice(self);
     }
 
-    #[cfg_attr(coverage_nightly, coverage(off))]
     fn read(buf: &[u8]) -> Self {
         buf.to_vec()
     }
 }
+
+impl Key for () {
+    fn weight(&self) -> usize {
+        0
+    }
+
+    fn serialized_len(&self) -> usize {
+        0
+    }
+
+    fn write(&self, _buf: &mut [u8]) {}
+
+    fn read(_buf: &[u8]) -> Self {}
+}
+
+impl Value for () {
+    fn weight(&self) -> usize {
+        0
+    }
+
+    fn serialized_len(&self) -> usize {
+        0
+    }
+
+    fn write(&self, _buf: &mut [u8]) {}
+
+    fn read(_buf: &[u8]) -> Self {}
+}
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 85dc087a..a529bba1 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -41,6 +41,7 @@ use foyer_storage::{
         rated_random::RatedRandomAdmissionPolicy, rated_ticket::RatedTicketAdmissionPolicy,
         AdmissionPolicy,
     },
+    compress::Compression,
     device::fs::FsDeviceConfig,
     error::Result,
     reinsertion::{
@@ -193,6 +194,10 @@ pub struct Args {
     /// use separate runtime
     #[arg(long, default_value_t = false)]
     runtime: bool,
+
+    /// available values: "none", "zstd"
+    #[arg(long, default_value = "none")]
+    compression: String,
 }
 
 #[derive(Debug)]
@@ -554,6 +559,12 @@ async fn main() {
         args.clean_region_threshold
     };
 
+    let compression = match args.compression.as_str() {
+        "none" => Compression::None,
+        "zstd" => Compression::Zstd,
+        _ => panic!("unsupported compression algorithm"),
+    };
+
     let config = LfuFsStoreConfig {
         name: "".to_string(),
         eviction_config,
@@ -572,6 +583,7 @@ async fn main() {
         recover_concurrency: args.recover_concurrency,
         allocation_timeout: Duration::from_millis(args.allocation_timeout as u64),
         clean_region_threshold,
+        compression,
     };
 
     let config = if args.runtime {
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index d99ceb29..2cb90693 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -32,6 +32,7 @@ thiserror = "1"
 tokio = { workspace = true }
 tracing = "0.1"
 twox-hash = "1"
+zstd = "0.13"
 
 [dev-dependencies]
 bytesize = "1"
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 789875ce..3a1146fb 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -12,11 +12,16 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use foyer_common::bits::{align_up, is_aligned};
+use foyer_common::{
+    bits::{align_up, is_aligned},
+    code::Key,
+};
 
 use crate::{
+    compress::Compression,
     device::{error::DeviceError, Device},
     flusher::Entry,
+    generic::{checksum, EntryHeader},
     region::{RegionHeader, RegionId, REGION_MAGIC},
 };
 
@@ -35,6 +40,7 @@ pub struct PositionedEntry {
     pub entry: Entry,
     pub region: RegionId,
     pub offset: usize,
+    pub len: usize,
 }
 
 #[derive(Debug)]
@@ -153,26 +159,160 @@ where
     ///
     /// Returns fully flushed entries if there is enough space in the current region.
     /// Otherwise, returns `NotEnough` error with the given `entry`.
-    pub async fn write(&mut self, entry: Entry) -> BufferResult<Vec<PositionedEntry>> {
-        // check region remaining size
-        let len = align_up(self.device.align(), entry.view.len());
-        if self.remaining() < len {
-            return Err(BufferError::NotEnough { entry });
+    ///
+    /// # Format
+    ///
+    /// | header | value (compressed) | key | <padding> |
+    pub async fn write<K: Key>(
+        &mut self,
+        Entry {
+            key,
+            key_len,
+            value_len,
+            sequence,
+            view,
+        }: Entry,
+        compression: Compression,
+    ) -> BufferResult<Vec<PositionedEntry>> {
+        debug_assert_eq!(view.len(), value_len);
+
+        if self.region.is_none() {
+            return Err(BufferError::NotEnough {
+                entry: Entry {
+                    key,
+                    key_len,
+                    value_len,
+                    sequence,
+                    view,
+                },
+            });
+        }
+
+        // reserve underlying vec capacity
+        let key = key.downcast::<K>().unwrap();
+        let uncompressed = align_up(
+            self.device.align(),
+            EntryHeader::serialized_len() + key_len + value_len,
+        );
+        self.buffer.reserve_exact(uncompressed);
+
+        // TODO(MrCroxx): skip if remaining is too small
+
+        // 1. try compress and write first
+        let old = self.buffer.len();
+        debug_assert!(is_aligned(self.device.align(), old));
+
+        match compression {
+            Compression::None => {
+                // early return if remaining size is not enough
+                if self.remaining() < uncompressed {
+                    return Err(BufferError::NotEnough {
+                        entry: Entry {
+                            key,
+                            key_len,
+                            value_len,
+                            sequence,
+                            view,
+                        },
+                    });
+                }
+                let mut cursor = self.buffer.len();
+                unsafe { self.buffer.set_len(cursor + uncompressed) };
+                debug_assert!(is_aligned(self.device.align(), self.buffer.len()));
+
+                // reserve space for header
+                cursor += EntryHeader::serialized_len();
+
+                // write value
+                std::io::copy(
+                    &mut &view[..],
+                    &mut &mut self.buffer[cursor..cursor + value_len],
+                )
+                .map_err(DeviceError::from)?;
+                debug_assert_eq!(&view[..], &self.buffer[cursor..cursor + value_len]);
+
+                // write key
+                cursor += value_len;
+                key.write(&mut self.buffer[cursor..cursor + key_len]);
+
+                // calculate checksum
+                cursor -= value_len;
+                let checksum = checksum(&self.buffer[cursor..cursor + value_len + key_len]);
+
+                // write entry header
+                cursor -= EntryHeader::serialized_len();
+                let header = EntryHeader {
+                    key_len: key_len as u32,
+                    value_len: value_len as u32,
+                    sequence,
+                    compression,
+                    checksum,
+                };
+                header.write(&mut self.buffer[cursor..cursor + EntryHeader::serialized_len()]);
+            }
+            Compression::Zstd => {
+                // reserve space for header
+                let mut cursor = self.buffer.len() + EntryHeader::serialized_len();
+                unsafe { self.buffer.set_len(cursor) };
+
+                // write compressed value
+                zstd::stream::copy_encode(&mut &view[..], &mut self.buffer, 0)
+                    .map_err(DeviceError::from)?;
+                let value_len = self.buffer.len() - cursor;
+
+                // write key
+                cursor += value_len;
+                self.buffer.reserve_exact(key_len);
+                unsafe { self.buffer.set_len(cursor + key_len) };
+                key.write(&mut self.buffer[cursor..cursor + key_len]);
+
+                // calculate checksum
+                cursor -= value_len;
+                let checksum = checksum(&self.buffer[cursor..cursor + value_len + key_len]);
+
+                // write entry header
+                cursor -= EntryHeader::serialized_len();
+                let header = EntryHeader {
+                    key_len: key_len as u32,
+                    value_len: value_len as u32,
+                    sequence,
+                    compression,
+                    checksum,
+                };
+                header.write(&mut self.buffer[cursor..cursor + EntryHeader::serialized_len()]);
+            }
         }
 
-        // write view and padding
-        let region = self.region.unwrap();
-        let offset = self.offset + self.buffer.len();
-        let target_len = self.buffer.len() + len;
-        debug_assert!(is_aligned(self.device.align(), offset));
+        // 2. if size exceeds region limit, rollback write and return
+        if self.offset + self.buffer.len() > self.device.region_size() {
+            unsafe { self.buffer.set_len(old) };
+            return Err(BufferError::NotEnough {
+                entry: Entry {
+                    key,
+                    key_len,
+                    value_len,
+                    sequence,
+                    view,
+                },
+            });
+        }
+
+        // 3. align buffer size
+        let target = align_up(self.device.align(), self.buffer.len());
+        self.buffer.reserve_exact(target - self.buffer.len());
+        unsafe { self.buffer.set_len(target) }
 
-        self.buffer.reserve_exact(len);
-        std::io::copy(&mut &entry.view[..], &mut self.buffer).map_err(DeviceError::from)?;
-        unsafe { self.buffer.set_len(target_len) };
         self.entries.push(PositionedEntry {
-            entry,
-            region,
-            offset,
+            entry: Entry {
+                key,
+                key_len,
+                value_len,
+                sequence,
+                view,
+            },
+            region: self.region.unwrap(),
+            offset: self.offset + old,
+            len: self.buffer.len() - old,
         });
 
         // flush if buffer equals or exceeds device io size
@@ -201,11 +341,13 @@ mod tests {
     use super::*;
 
     fn ent(view: RingBufferView) -> Entry {
+        let key = Arc::new(());
+        let value_len = view.len();
         Entry {
-            key: Arc::new(()),
+            key,
             view,
             key_len: 0,
-            value_len: 0,
+            value_len,
             sequence: 0,
         }
     }
@@ -229,6 +371,8 @@ mod tests {
         let mut buffer = FlushBuffer::new(device.clone(), DEFAULT_BUFFER_CAPACITY);
         assert_eq!(buffer.region(), None);
 
+        const HEADER: usize = EntryHeader::serialized_len();
+
         {
             let view = {
                 let mut view = ring.allocate(5 * 1024 - 128, 0).await; // ~ 6 KiB
@@ -239,7 +383,7 @@ mod tests {
             let entry = ent(view);
             assert_eq!(ring.continuum(), 0);
 
-            let res = buffer.write(entry).await;
+            let res = buffer.write::<()>(entry, Compression::None).await;
             let entry = match res {
                 Err(BufferError::NotEnough { entry }) => entry,
                 _ => panic!("should be not enough error"),
@@ -249,16 +393,25 @@ mod tests {
             assert!(entries.is_empty());
 
             // 4 ~ 12 KiB
-            let entries = buffer.write(entry.clone()).await.unwrap();
+            let entries = buffer
+                .write::<()>(entry.clone(), Compression::None)
+                .await
+                .unwrap();
             assert!(entries.is_empty());
             // 12 ~ 20 KiB
-            let entries = buffer.write(entry.clone()).await.unwrap();
+            let entries = buffer
+                .write::<()>(entry.clone(), Compression::None)
+                .await
+                .unwrap();
             assert_eq!(entries.len(), 2);
             assert_eq!(entries[0].offset, 4 * 1024);
             assert_eq!(entries[1].offset, 12 * 1024);
 
             // 20 ~ 28 KiB
-            let entries = buffer.write(entry.clone()).await.unwrap();
+            let entries = buffer
+                .write::<()>(entry.clone(), Compression::None)
+                .await
+                .unwrap();
             assert!(entries.is_empty());
             let entries = buffer.flush().await.unwrap();
             assert_eq!(entries.len(), 1);
@@ -267,9 +420,18 @@ mod tests {
             let buf = device.io_buffer(64 * 1024, 64 * 1024);
             let (res, buf) = device.read(buf, .., 0, 0).await;
             res.unwrap();
-            assert_eq!(&buf[4 * 1024..9 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
-            assert_eq!(&buf[12 * 1024..17 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
-            assert_eq!(&buf[20 * 1024..25 * 1024 - 128], &[b'x'; 5 * 1024 - 128]);
+            assert_eq!(
+                &buf[HEADER + 4 * 1024..HEADER + 9 * 1024 - 128],
+                &[b'x'; 5 * 1024 - 128]
+            );
+            assert_eq!(
+                &buf[HEADER + 12 * 1024..HEADER + 17 * 1024 - 128],
+                &[b'x'; 5 * 1024 - 128]
+            );
+            assert_eq!(
+                &buf[HEADER + 20 * 1024..HEADER + 25 * 1024 - 128],
+                &[b'x'; 5 * 1024 - 128]
+            );
 
             assert!(buffer.entries.is_empty());
         }
@@ -286,7 +448,7 @@ mod tests {
             };
             let entry = ent(view);
 
-            let res = buffer.write(entry).await;
+            let res = buffer.write::<()>(entry, Compression::None).await;
             let entry = match res {
                 Err(BufferError::NotEnough { entry }) => entry,
                 _ => panic!("should be not enough error"),
@@ -296,27 +458,34 @@ mod tests {
             assert!(entries.is_empty());
 
             // 4 ~ 60 KiB
-            let entries = buffer.write(entry).await.unwrap();
+            let entries = buffer.write::<()>(entry, Compression::None).await.unwrap();
             assert_eq!(entries.len(), 1);
             assert_eq!(entries[0].offset, 4 * 1024);
 
             let view = {
                 let mut view = ring.allocate(3 * 1024 - 128, 2).await; // ~ 3 KiB
                 (&mut view[..]).put_slice(&[b'x'; 3 * 1024 - 128]);
+                view.shrink_to(3 * 1024 - 128);
                 view.freeze()
             };
             let entry = ent(view);
 
             // 60 ~ 64 KiB
-            let entries = buffer.write(entry).await.unwrap();
+            let entries = buffer.write::<()>(entry, Compression::None).await.unwrap();
             assert_eq!(entries.len(), 1);
             assert_eq!(entries[0].offset, 60 * 1024);
 
             let buf = device.io_buffer(64 * 1024, 64 * 1024);
             let (res, buf) = device.read(buf, .., 1, 0).await;
             res.unwrap();
-            assert_eq!(&buf[4 * 1024..58 * 1024 - 128], &[b'x'; 54 * 1024 - 128]);
-            assert_eq!(&buf[60 * 1024..63 * 1024 - 128], &[b'x'; 3 * 1024 - 128]);
+            assert_eq!(
+                &buf[HEADER + 4 * 1024..HEADER + 58 * 1024 - 128],
+                &[b'x'; 54 * 1024 - 128]
+            );
+            assert_eq!(
+                &buf[HEADER + 60 * 1024..HEADER + 63 * 1024 - 128],
+                &[b'x'; 3 * 1024 - 128]
+            );
 
             assert!(buffer.entries.is_empty());
         }
diff --git a/foyer-storage/src/compress.rs b/foyer-storage/src/compress.rs
new file mode 100644
index 00000000..268bf425
--- /dev/null
+++ b/foyer-storage/src/compress.rs
@@ -0,0 +1,38 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// TODO(MrCroxx): unify compress interface?
+
+#[derive(Debug, Clone, Copy)]
+pub enum Compression {
+    None,
+    Zstd,
+}
+
+impl Compression {
+    pub fn to_u8(&self) -> u8 {
+        match self {
+            Self::None => 0,
+            Self::Zstd => 1,
+        }
+    }
+
+    pub fn try_from_u8(v: u8) -> Option<Self> {
+        match v {
+            0 => Some(Self::None),
+            1 => Some(Self::Zstd),
+            _ => None,
+        }
+    }
+}
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 3df3c86a..e8f85d04 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -15,6 +15,7 @@
 use crate::{
     buffer::{BufferError, FlushBuffer, PositionedEntry},
     catalog::{Catalog, Index, Item, Sequence},
+    compress::Compression,
     device::Device,
     error::{Error, Result},
     metrics::Metrics,
@@ -73,6 +74,8 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
+    compression: Compression,
+
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
     catalog: Arc<Catalog<K>>,
@@ -93,8 +96,10 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
+    #[expect(clippy::too_many_arguments)]
     pub fn new(
         default_buffer_capacity: usize,
+        compression: Compression,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         catalog: Arc<Catalog<K>>,
         device: D,
@@ -104,6 +109,7 @@ where
     ) -> Self {
         let buffer = FlushBuffer::new(device.clone(), default_buffer_capacity);
         Self {
+            compression,
             region_manager,
             catalog,
             buffer,
@@ -139,7 +145,7 @@ where
 
         let old_region = self.buffer.region();
 
-        let entry = match self.buffer.write(entry).await {
+        let entry = match self.buffer.write::<K>(entry, self.compression).await {
             Err(BufferError::NotEnough { entry }) => entry,
 
             Ok(entries) => return self.update_catalog(entries).await,
@@ -176,7 +182,7 @@ where
         );
 
         // 3. retry write
-        let entries = self.buffer.write(entry).await?;
+        let entries = self.buffer.write::<K>(entry, self.compression).await?;
         self.update_catalog(entries).await?;
 
         drop(timer);
@@ -185,25 +191,30 @@ where
 
     #[tracing::instrument(skip(self))]
     async fn update_catalog(&self, entries: Vec<PositionedEntry>) -> Result<()> {
+        // record fully flushed bytes by the way
+        let mut bytes = 0;
+
         let timer = self.metrics.inner_op_duration_update_catalog.start_timer();
         for PositionedEntry {
             entry:
                 Entry {
                     key,
-                    view,
+                    view: _,
                     key_len,
                     value_len,
                     sequence,
                 },
             region,
             offset,
+            len,
         } in entries
         {
+            bytes += len;
             let key = key.downcast::<K>().unwrap();
             let index = Index::Region {
                 view: self.region_manager.region(&region).view(
                     offset as u32,
-                    view.aligned() as u32,
+                    len as u32,
                     key_len as u32,
                     value_len as u32,
                 ),
@@ -212,6 +223,9 @@ where
             self.catalog.insert(key, item);
         }
         drop(timer);
+
+        self.metrics.op_bytes_flush.inc_by(bytes as u64);
+
         Ok(())
     }
 }
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 09d51299..22238568 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -38,6 +38,7 @@ use twox_hash::XxHash64;
 use crate::{
     admission::AdmissionPolicy,
     catalog::{Catalog, Index, Item, Sequence},
+    compress::Compression,
     device::Device,
     error::Result,
     flusher::{Entry, Flusher},
@@ -120,6 +121,9 @@ where
 
     /// Concurrency of recovery.
     pub recover_concurrency: usize,
+
+    /// Compression algorithm.
+    pub compression: Compression,
 }
 
 impl<K, V, D, EP> Debug for GenericStoreConfig<K, V, D, EP>
@@ -147,6 +151,7 @@ where
             .field("allocation_timeout", &self.allocation_timeout)
             .field("clean_region_threshold", &self.clean_region_threshold)
             .field("recover_concurrency", &self.recover_concurrency)
+            .field("compression", &self.compression)
             .finish()
     }
 }
@@ -177,6 +182,7 @@ where
             allocation_timeout: self.allocation_timeout,
             clean_region_threshold: self.clean_region_threshold,
             recover_concurrency: self.recover_concurrency,
+            compression: self.compression,
         }
     }
 }
@@ -339,6 +345,7 @@ where
             .map(|(stop_rx, entry_rx)| {
                 Flusher::new(
                     config.flusher_buffer_size,
+                    config.compression,
                     region_manager.clone(),
                     catalog.clone(),
                     device.clone(),
@@ -431,21 +438,14 @@ where
 
         match index {
             crate::catalog::Index::RingBuffer { view } => {
-                let res = match read_entry::<K, V>(view.as_ref()) {
-                    Some((_key, value)) => Ok(Some(value)),
-                    None => {
-                        // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
-                        self.inner.catalog.remove(key);
-                        Ok(None)
-                    }
-                };
+                let v = V::read(&view);
 
                 self.inner
                     .metrics
                     .op_duration_lookup_hit
                     .observe(now.elapsed().as_secs_f64());
 
-                res
+                Ok(Some(v))
             }
             // read from region
             crate::catalog::Index::Region { view } => {
@@ -518,12 +518,6 @@ where
         &self.inner.reinsertions
     }
 
-    fn serialized_len(&self, key: &K, value: &V) -> usize {
-        let unaligned =
-            EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len();
-        bits::align_up(self.inner.device.align(), unaligned)
-    }
-
     #[tracing::instrument(skip(self))]
     async fn recover(&self, concurrency: usize) -> Result<Sequence> {
         tracing::info!("start store recovery");
@@ -629,27 +623,22 @@ where
             admission.on_insert(&key, writer.weight, &self.inner.metrics, judge);
         }
 
-        let serialized_len = self.serialized_len(&key, &value);
+        // only write value to ring buffer
+        let len = bits::align_up(self.inner.device.align(), value.serialized_len());
 
-        if key.serialized_len() + value.serialized_len() != writer.weight {
-            tracing::error!(
-                "weight != key.serialized_len() + value.serialized_len(), weight: {}, key size: {}, value size: {}, key: {:?}",
-                writer.weight, key.serialized_len(), value.serialized_len(), key
-            );
-        }
-
-        self.inner
-            .metrics
-            .op_bytes_insert
-            .inc_by(serialized_len as u64);
+        // record aligned header + key + value size for metrics
+        self.inner.metrics.op_bytes_insert.inc_by(bits::align_up(
+            self.inner.device.align(),
+            EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len(),
+        ) as u64);
 
         self.inner
             .metrics
             .inner_bytes_ring_buffer_remains
             .set(self.inner.ring.remains() as i64);
-        let mut view = self.inner.ring.allocate(serialized_len, sequence).await;
-        let written = write_entry(&mut view, &key, &value, sequence);
-        view.shrink_to(written);
+        let mut view = self.inner.ring.allocate(len, sequence).await;
+        value.write(&mut view);
+        view.shrink_to(value.serialized_len());
         let view = view.freeze();
 
         let key = Arc::new(key);
@@ -820,79 +809,57 @@ where
 //     }
 // }
 
-const ENTRY_MAGIC: u32 = 0x97_00_00_00;
-const ENTRY_MAGIC_MASK: u32 = 0xFF_00_00_00;
+const ENTRY_MAGIC: u32 = 0x97_03_27_00;
+const ENTRY_MAGIC_MASK: u32 = 0xFF_FF_FF_00;
 
 #[derive(Debug)]
-struct EntryHeader {
-    key_len: u32,
-    value_len: u32,
-    sequence: Sequence,
-    checksum: u64,
+pub struct EntryHeader {
+    pub key_len: u32,
+    pub value_len: u32,
+    pub sequence: Sequence,
+    pub checksum: u64,
+    pub compression: Compression,
 }
 
 impl EntryHeader {
-    fn serialized_len() -> usize {
-        4 + 4 + 8 + 8
+    pub const fn serialized_len() -> usize {
+        4 + 4 + 8 + 8 + 4 /* magic & compression */
     }
 
-    fn write(&self, mut buf: &mut [u8]) {
-        buf.put_u32(self.key_len | ENTRY_MAGIC);
+    pub fn write(&self, mut buf: &mut [u8]) {
+        buf.put_u32(self.key_len);
         buf.put_u32(self.value_len);
         buf.put_u64(self.sequence);
         buf.put_u64(self.checksum);
+
+        let v = ENTRY_MAGIC | self.compression.to_u8() as u32;
+        buf.put_u32(v);
     }
 
-    fn read(mut buf: &[u8]) -> Option<Self> {
-        let head = buf.get_u32();
-        let magic = head & ENTRY_MAGIC_MASK;
+    pub fn read(mut buf: &[u8]) -> Option<Self> {
+        let key_len = buf.get_u32();
+        let value_len = buf.get_u32();
+        let sequence = buf.get_u64();
+        let checksum = buf.get_u64();
 
+        let v = buf.get_u32();
+        let magic = v & ENTRY_MAGIC_MASK;
         if magic != ENTRY_MAGIC {
             return None;
         }
-
-        let key_len = head ^ ENTRY_MAGIC;
-        let value_len = buf.get_u32();
-        let sequence = buf.get_u64();
-        let checksum = buf.get_u64();
+        let compression = Compression::try_from_u8(v as u8)?;
 
         Some(Self {
             key_len,
             value_len,
             sequence,
+            compression,
             checksum,
         })
     }
 }
 
-/// | header | value | key | <padding> |
-///
-/// # Safety
-///
-/// `buf.len()` must excatly fit entry size
-fn write_entry<K, V>(buf: &mut [u8], key: &K, value: &V, sequence: Sequence) -> usize
-where
-    K: Key,
-    V: Value,
-{
-    let mut offset = EntryHeader::serialized_len();
-    value.write(&mut buf[offset..offset + value.serialized_len()]);
-    offset += value.serialized_len();
-    key.write(&mut buf[offset..offset + key.serialized_len()]);
-    offset += key.serialized_len();
-    let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
-
-    let header = EntryHeader {
-        key_len: key.serialized_len() as u32,
-        value_len: value.serialized_len() as u32,
-        sequence,
-        checksum,
-    };
-    header.write(&mut buf[..EntryHeader::serialized_len()]);
-    offset
-}
-
-/// | header | value | key | <padding> |
+/// | header | value (compressed) | key | <padding> |
 ///
 /// # Safety
 ///
@@ -902,11 +869,27 @@ where
     K: Key,
     V: Value,
 {
+    // read entry header
     let header = EntryHeader::read(buf)?;
 
+    // read value
     let mut offset = EntryHeader::serialized_len();
-    let value = V::read(&buf[offset..offset + header.value_len as usize]);
+    let compressed = &buf[offset..offset + header.value_len as usize];
     offset += header.value_len as usize;
+    let value = match header.compression {
+        Compression::None => V::read(compressed),
+        Compression::Zstd => {
+            let mut decompressed =
+                Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
+            if let Err(e) = zstd::stream::copy_decode(&mut &compressed[..], &mut decompressed) {
+                tracing::warn!("decompress error: {}", e);
+                return None;
+            }
+            V::read(&decompressed[..])
+        }
+    };
+
+    // read key
     let key = K::read(&buf[offset..offset + header.key_len as usize]);
     offset += header.key_len as usize;
 
@@ -923,7 +906,7 @@ where
     Some((key, value))
 }
 
-fn checksum(buf: &[u8]) -> u64 {
+pub fn checksum(buf: &[u8]) -> u64 {
     let mut hasher = XxHash64::with_seed(0);
     hasher.write(buf);
     hasher.finish()
@@ -1205,6 +1188,7 @@ mod tests {
             recover_concurrency: 2,
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
+            compression: Compression::None,
         };
 
         let store = TestStore::open(config).await.unwrap();
@@ -1260,6 +1244,7 @@ mod tests {
             recover_concurrency: 2,
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
+            compression: Compression::None,
         };
         let store = TestStore::open(config).await.unwrap();
 
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 3bad82f0..b68dc682 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -252,6 +252,7 @@ mod tests {
             recover_concurrency: 2,
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
+            compression: crate::compress::Compression::None,
         };
 
         let (store, handle) = LazyStorage::<_, _, Store<_, _>>::with_handle(config.into());
@@ -290,6 +291,7 @@ mod tests {
             recover_concurrency: 2,
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
+            compression: crate::compress::Compression::None,
         };
 
         let (store, handle) = LazyStorage::<_, _, Store<_, _>>::with_handle(config.into());
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 24089840..762dd528 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -25,6 +25,7 @@
 pub mod admission;
 pub mod buffer;
 pub mod catalog;
+pub mod compress;
 pub mod device;
 pub mod error;
 pub mod flusher;
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 0b707148..86d96adc 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -17,6 +17,7 @@
 
 use foyer_intrusive::eviction::fifo::FifoConfig;
 use foyer_storage::{
+    compress::Compression,
     device::fs::FsDeviceConfig,
     lazy::LazyStore,
     runtime::{RuntimeConfig, RuntimeLazyStore, RuntimeStorageConfig, RuntimeStore},
@@ -142,6 +143,41 @@ async fn test_store() {
         allocation_timeout: Duration::from_millis(10),
         clean_region_threshold: 1,
         recover_concurrency: 2,
+        compression: Compression::None,
+    };
+
+    test_storage::<Store<_, _>>(config.into(), recorder).await;
+}
+
+#[tokio::test]
+async fn test_store_zstd() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let recorder = Arc::new(JudgeRecorder::default());
+    let config = FifoFsStoreConfig {
+        name: "".to_string(),
+        eviction_config: FifoConfig,
+        device_config: FsDeviceConfig {
+            dir: PathBuf::from(tempdir.path()),
+            capacity: 4 * MB,
+            file_capacity: 1 * MB,
+            align: 4 * KB,
+            io_size: 4 * KB,
+        },
+        allocator_bits: 0,
+        ring_buffer_capacity: 16 * MB,
+        catalog_bits: 1,
+        admissions: vec![recorder.clone()],
+        reinsertions: vec![recorder.clone()],
+        buffer_pool_size: 2 * MB,
+        flusher_buffer_size: 0,
+        flushers: 1,
+        flush_rate_limit: 0,
+        reclaimers: 1,
+        reclaim_rate_limit: 0,
+        allocation_timeout: Duration::from_millis(10),
+        clean_region_threshold: 1,
+        recover_concurrency: 2,
+        compression: Compression::Zstd,
     };
 
     test_storage::<Store<_, _>>(config.into(), recorder).await;
@@ -175,6 +211,7 @@ async fn test_lazy_store() {
         allocation_timeout: Duration::from_millis(10),
         clean_region_threshold: 1,
         recover_concurrency: 2,
+        compression: Compression::None,
     };
 
     test_storage::<LazyStore<_, _>>(config.into(), recorder).await;
@@ -209,6 +246,7 @@ async fn test_runtime_store() {
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             recover_concurrency: 2,
+            compression: Compression::None,
         }
         .into(),
         runtime: RuntimeConfig {
@@ -249,6 +287,7 @@ async fn test_runtime_lazy_store() {
             allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             recover_concurrency: 2,
+            compression: Compression::None,
         }
         .into(),
         runtime: RuntimeConfig {
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index ab2c5188..bf0c75bf 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -35,6 +35,7 @@ tracing = { version = "0.1" }
 tracing-core = { version = "0.1" }
 
 [build-dependencies]
+cc = { version = "1", default-features = false, features = ["parallel"] }
 either = { version = "1", default-features = false, features = ["use_std"] }
 itertools = { version = "0.11" }
 proc-macro2 = { version = "1" }

From 9232b3aa468460b60c44c26eac62dadf3594a383 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 1 Nov 2023 17:42:51 +0800
Subject: [PATCH 163/261] chore: cleanup config (#199)

* chore: cleanup config

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: cleanup bench tool args

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml       |  4 +-
 .github/workflows/main.yml          |  4 +-
 .github/workflows/pull-request.yml  |  4 +-
 foyer-storage-bench/src/main.rs     | 23 ----------
 foyer-storage/src/generic.rs        | 69 +++++++----------------------
 foyer-storage/src/lazy.rs           | 10 +----
 foyer-storage/src/region_manager.rs | 14 +-----
 foyer-storage/tests/storage_test.rs | 20 ---------
 8 files changed, 25 insertions(+), 123 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 93d163e1..fb14a4f1 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -120,7 +120,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -149,7 +149,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ea39db0c..a852da33 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -127,7 +127,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -156,7 +156,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 2d88b355..5dfb8d47 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -126,7 +126,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -155,7 +155,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --allocator-bits 1 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index a529bba1..0708f87c 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -108,10 +108,6 @@ pub struct Args {
     #[arg(long, default_value_t = 64)]
     region_size: usize,
 
-    /// (MiB)
-    #[arg(long, default_value_t = 1024)]
-    buffer_pool_size: usize,
-
     #[arg(long, default_value_t = 0)]
     flusher_buffer_size: usize,
 
@@ -160,29 +156,14 @@ pub struct Args {
     #[arg(long, default_value_t = 0)]
     ticket_reinsert_rate_limit: usize,
 
-    /// (MiB/s)
-    #[arg(long, default_value_t = 0)]
-    flush_rate_limit: usize,
-
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     reclaim_rate_limit: usize,
 
-    /// (ms)
-    #[arg(long, default_value_t = 10)]
-    allocation_timeout: usize,
-
     /// `0` means equal to reclaimer count
     #[arg(long, default_value_t = 0)]
     clean_region_threshold: usize,
 
-    /// the count of allocators is `2 ^ allocator bits`
-    ///
-    /// Note: The count of allocators should be greater than buffer count.
-    ///       (buffer count = buffer pool size / device region size)
-    #[arg(long, default_value_t = 0)]
-    allocator_bits: usize,
-
     /// Catalog indices sharding bits.
     #[arg(long, default_value_t = 6)]
     catalog_bits: usize,
@@ -569,19 +550,15 @@ async fn main() {
         name: "".to_string(),
         eviction_config,
         device_config,
-        allocator_bits: args.allocator_bits,
         ring_buffer_capacity: args.ring_buffer_capacity * 1024 * 1024,
         catalog_bits: args.catalog_bits,
         admissions,
         reinsertions,
-        buffer_pool_size: args.buffer_pool_size * 1024 * 1024,
         flusher_buffer_size: args.flusher_buffer_size,
         flushers: args.flushers,
-        flush_rate_limit: args.flush_rate_limit * 1024 * 1024,
         reclaimers: args.reclaimers,
         reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
-        allocation_timeout: Duration::from_millis(args.allocation_timeout as u64),
         clean_region_threshold,
         compression,
     };
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 22238568..d1aa8cea 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -75,12 +75,6 @@ where
     /// Device configurations.
     pub device_config: D::Config,
 
-    /// The count of allocators is `2 ^ allocator bits`.
-    ///
-    /// Note: The count of allocators should be greater than buffer count.
-    ///       (buffer count = buffer pool size / device region size)
-    pub allocator_bits: usize,
-
     /// `ring_buffer_capacity` will be aligned up to device align.
     pub ring_buffer_capacity: usize,
 
@@ -93,27 +87,18 @@ where
     /// Reinsertion policies.
     pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
 
-    /// Buffer pool size, should be a multiplier of device region size.
-    pub buffer_pool_size: usize,
-
     /// Flusher default buffer capacity, must be equals or larger than device io size.
     pub flusher_buffer_size: usize,
 
     /// Count of flushers.
     pub flushers: usize,
 
-    /// Flush rate limits.
-    pub flush_rate_limit: usize,
-
     /// Count of reclaimers.
     pub reclaimers: usize,
 
     /// Flush rate limits.
     pub reclaim_rate_limit: usize,
 
-    /// Allocation timout for skippable writers.
-    pub allocation_timeout: Duration,
-
     /// Clean region count threshold to trigger reclamation.
     ///
     /// `clean_region_threshold` is recommended to be equal or larger than `reclaimers`.
@@ -137,18 +122,14 @@ where
         f.debug_struct("StoreConfig")
             .field("eviction_config", &self.eviction_config)
             .field("device_config", &self.device_config)
-            .field("allocator_bits", &self.allocator_bits)
             .field("ring_buffer_capacity", &self.ring_buffer_capacity)
             .field("catalog_bits", &self.catalog_bits)
             .field("admissions", &self.admissions)
             .field("reinsertions", &self.reinsertions)
-            .field("buffer_pool_size", &self.buffer_pool_size)
             .field("flusher_buffer_size", &self.flusher_buffer_size)
             .field("flushers", &self.flushers)
-            .field("flush_rate_limit", &self.flush_rate_limit)
             .field("reclaimers", &self.reclaimers)
             .field("reclaim_rate_limit", &self.reclaim_rate_limit)
-            .field("allocation_timeout", &self.allocation_timeout)
             .field("clean_region_threshold", &self.clean_region_threshold)
             .field("recover_concurrency", &self.recover_concurrency)
             .field("compression", &self.compression)
@@ -168,18 +149,14 @@ where
             name: self.name.clone(),
             eviction_config: self.eviction_config.clone(),
             device_config: self.device_config.clone(),
-            allocator_bits: self.allocator_bits,
             ring_buffer_capacity: self.ring_buffer_capacity,
             catalog_bits: self.catalog_bits,
             admissions: self.admissions.clone(),
             reinsertions: self.reinsertions.clone(),
-            buffer_pool_size: self.buffer_pool_size,
             flusher_buffer_size: self.flusher_buffer_size,
             flushers: self.flushers,
-            flush_rate_limit: self.flush_rate_limit,
             reclaimers: self.reclaimers,
             reclaim_rate_limit: self.reclaim_rate_limit,
-            allocation_timeout: self.allocation_timeout,
             clean_region_threshold: self.clean_region_threshold,
             recover_concurrency: self.recover_concurrency,
             compression: self.compression,
@@ -262,15 +239,6 @@ where
         let device = D::open(config.device_config).await?;
         assert!(device.regions() >= config.flushers * 2);
 
-        let buffer_count = config.buffer_pool_size / device.region_size();
-
-        if buffer_count < (1 << config.allocator_bits) {
-            return Err(anyhow::anyhow!(
-                "The count of allocators shoule be greater than buffer count."
-            )
-            .into());
-        }
-
         let ring = Arc::new(RingBuffer::with_metrics_in(
             device.align(),
             config.ring_buffer_capacity,
@@ -279,7 +247,6 @@ where
         ));
 
         let region_manager = Arc::new(RegionManager::new(
-            buffer_count,
             device.regions(),
             config.eviction_config,
             device.clone(),
@@ -438,14 +405,19 @@ where
 
         match index {
             crate::catalog::Index::RingBuffer { view } => {
-                let v = V::read(&view);
+                let value = V::read(&view);
 
                 self.inner
                     .metrics
                     .op_duration_lookup_hit
                     .observe(now.elapsed().as_secs_f64());
 
-                Ok(Some(v))
+                self.inner
+                    .metrics
+                    .op_bytes_lookup
+                    .inc_by(value.serialized_len() as u64);
+
+                Ok(Some(value))
             }
             // read from region
             crate::catalog::Index::Region { view } => {
@@ -455,8 +427,8 @@ where
                 let region = self.inner.region_manager.region(region);
 
                 // TODO(MrCroxx): read value only
-                let slice = match region.load(view).await? {
-                    Some(slice) => slice,
+                let buf = match region.load(view).await? {
+                    Some(buf) => buf,
                     None => {
                         // Remove index if the storage layer fails to lookup it (because of region version mismatch).
                         self.inner.catalog.remove(key);
@@ -468,13 +440,14 @@ where
                     }
                 };
 
-                self.inner
-                    .metrics
-                    .op_bytes_lookup
-                    .inc_by(slice.len() as u64);
-
-                let res = match read_entry::<K, V>(slice.as_ref()) {
-                    Some((_key, value)) => Ok(Some(value)),
+                let res = match read_entry::<K, V>(buf.as_ref()) {
+                    Some((_key, value)) => {
+                        self.inner
+                            .metrics
+                            .op_bytes_lookup
+                            .inc_by(value.serialized_len() as u64);
+                        Ok(Some(value))
+                    }
                     None => {
                         // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
                         self.inner.catalog.remove(key);
@@ -1174,19 +1147,15 @@ mod tests {
                 align: 4 * KB,
                 io_size: 4 * KB,
             },
-            allocator_bits: 1,
             ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions,
             reinsertions,
-            buffer_pool_size: 8 * MB,
             flusher_buffer_size: 0,
             flushers: 1,
-            flush_rate_limit: 0,
             reclaimers: 1,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             compression: Compression::None,
         };
@@ -1230,19 +1199,15 @@ mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            allocator_bits: 1,
             ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
-            buffer_pool_size: 8 * MB,
             flusher_buffer_size: 0,
             flushers: 1,
-            flush_rate_limit: 0,
             reclaimers: 0,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             compression: Compression::None,
         };
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index b68dc682..39e5a746 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -209,7 +209,7 @@ pub type LazyStoreWriter<K, V> = LazyStorageWriter<K, V, Store<K, V>>;
 
 #[cfg(test)]
 mod tests {
-    use std::{path::PathBuf, time::Duration};
+    use std::path::PathBuf;
 
     use foyer_intrusive::eviction::fifo::FifoConfig;
 
@@ -238,19 +238,15 @@ mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            allocator_bits: 1,
             ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
-            buffer_pool_size: 8 * MB,
             flusher_buffer_size: 0,
             flushers: 1,
-            flush_rate_limit: 0,
             reclaimers: 1,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             compression: crate::compress::Compression::None,
         };
@@ -277,19 +273,15 @@ mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            allocator_bits: 1,
             ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
-            buffer_pool_size: 8 * MB,
             flusher_buffer_size: 0,
             flushers: 1,
-            flush_rate_limit: 0,
             reclaimers: 1,
             reclaim_rate_limit: 0,
             recover_concurrency: 2,
-            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             compression: crate::compress::Compression::None,
         };
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 057b760a..dc691a4c 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -63,19 +63,7 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    pub fn new(
-        buffer_count: usize,
-        region_count: usize,
-        eviction_config: EP::Config,
-        device: D,
-    ) -> Self {
-        let buffers = AsyncQueue::new();
-        for _ in 0..buffer_count {
-            let len = device.region_size();
-            let buffer = device.io_buffer(len, len);
-            buffers.release(buffer);
-        }
-
+    pub fn new(region_count: usize, eviction_config: EP::Config, device: D) -> Self {
         let eviction = EP::new(eviction_config);
         let clean_regions = AsyncQueue::new();
 
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 86d96adc..3887e261 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -129,18 +129,14 @@ async fn test_store() {
             align: 4 * KB,
             io_size: 4 * KB,
         },
-        allocator_bits: 0,
         ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
-        buffer_pool_size: 2 * MB,
         flusher_buffer_size: 0,
         flushers: 1,
-        flush_rate_limit: 0,
         reclaimers: 1,
         reclaim_rate_limit: 0,
-        allocation_timeout: Duration::from_millis(10),
         clean_region_threshold: 1,
         recover_concurrency: 2,
         compression: Compression::None,
@@ -163,18 +159,14 @@ async fn test_store_zstd() {
             align: 4 * KB,
             io_size: 4 * KB,
         },
-        allocator_bits: 0,
         ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
-        buffer_pool_size: 2 * MB,
         flusher_buffer_size: 0,
         flushers: 1,
-        flush_rate_limit: 0,
         reclaimers: 1,
         reclaim_rate_limit: 0,
-        allocation_timeout: Duration::from_millis(10),
         clean_region_threshold: 1,
         recover_concurrency: 2,
         compression: Compression::Zstd,
@@ -197,18 +189,14 @@ async fn test_lazy_store() {
             align: 4 * KB,
             io_size: 4 * KB,
         },
-        allocator_bits: 0,
         ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
-        buffer_pool_size: 2 * MB,
         flusher_buffer_size: 0,
         flushers: 1,
-        flush_rate_limit: 0,
         reclaimers: 1,
         reclaim_rate_limit: 0,
-        allocation_timeout: Duration::from_millis(10),
         clean_region_threshold: 1,
         recover_concurrency: 2,
         compression: Compression::None,
@@ -232,18 +220,14 @@ async fn test_runtime_store() {
                 align: 4 * KB,
                 io_size: 4 * KB,
             },
-            allocator_bits: 0,
             ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
-            buffer_pool_size: 2 * MB,
             flusher_buffer_size: 0,
             flushers: 1,
-            flush_rate_limit: 0,
             reclaimers: 1,
             reclaim_rate_limit: 0,
-            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             recover_concurrency: 2,
             compression: Compression::None,
@@ -273,18 +257,14 @@ async fn test_runtime_lazy_store() {
                 align: 4 * KB,
                 io_size: 4 * KB,
             },
-            allocator_bits: 0,
             ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
-            buffer_pool_size: 2 * MB,
             flusher_buffer_size: 0,
             flushers: 1,
-            flush_rate_limit: 0,
             reclaimers: 1,
             reclaim_rate_limit: 0,
-            allocation_timeout: Duration::from_millis(10),
             clean_region_threshold: 1,
             recover_concurrency: 2,
             compression: Compression::None,

From 0ba922eb71127a06544f58308c230ab4c455073d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 3 Nov 2023 18:32:35 +0800
Subject: [PATCH 164/261] chore: clear compression code logic (#201)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/buffer.rs   | 128 +++++++++++-----------------------
 foyer-storage/src/compress.rs |   2 +-
 foyer-storage/src/flusher.rs  |  18 ++---
 foyer-storage/src/generic.rs  |   9 +--
 foyer-storage/src/region.rs   |  16 +----
 5 files changed, 49 insertions(+), 124 deletions(-)

diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 3a1146fb..fb2b53f8 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -167,21 +167,15 @@ where
         &mut self,
         Entry {
             key,
-            key_len,
-            value_len,
             sequence,
             view,
         }: Entry,
         compression: Compression,
     ) -> BufferResult<Vec<PositionedEntry>> {
-        debug_assert_eq!(view.len(), value_len);
-
         if self.region.is_none() {
             return Err(BufferError::NotEnough {
                 entry: Entry {
                     key,
-                    key_len,
-                    value_len,
                     sequence,
                     view,
                 },
@@ -192,7 +186,7 @@ where
         let key = key.downcast::<K>().unwrap();
         let uncompressed = align_up(
             self.device.align(),
-            EntryHeader::serialized_len() + key_len + value_len,
+            EntryHeader::serialized_len() + key.serialized_len() + view.len(),
         );
         self.buffer.reserve_exact(uncompressed);
 
@@ -202,86 +196,55 @@ where
         let old = self.buffer.len();
         debug_assert!(is_aligned(self.device.align(), old));
 
+        if compression == Compression::None && self.remaining() < uncompressed {
+            // early return if remaining size is not enough
+            return Err(BufferError::NotEnough {
+                entry: Entry {
+                    key,
+                    sequence,
+                    view,
+                },
+            });
+        }
+
+        let mut cursor = self.buffer.len();
+
+        // reserve space for header
+        cursor += EntryHeader::serialized_len();
+        unsafe { self.buffer.set_len(cursor) };
+
         match compression {
             Compression::None => {
-                // early return if remaining size is not enough
-                if self.remaining() < uncompressed {
-                    return Err(BufferError::NotEnough {
-                        entry: Entry {
-                            key,
-                            key_len,
-                            value_len,
-                            sequence,
-                            view,
-                        },
-                    });
-                }
-                let mut cursor = self.buffer.len();
-                unsafe { self.buffer.set_len(cursor + uncompressed) };
-                debug_assert!(is_aligned(self.device.align(), self.buffer.len()));
-
-                // reserve space for header
-                cursor += EntryHeader::serialized_len();
-
-                // write value
-                std::io::copy(
-                    &mut &view[..],
-                    &mut &mut self.buffer[cursor..cursor + value_len],
-                )
-                .map_err(DeviceError::from)?;
-                debug_assert_eq!(&view[..], &self.buffer[cursor..cursor + value_len]);
-
-                // write key
-                cursor += value_len;
-                key.write(&mut self.buffer[cursor..cursor + key_len]);
-
-                // calculate checksum
-                cursor -= value_len;
-                let checksum = checksum(&self.buffer[cursor..cursor + value_len + key_len]);
-
-                // write entry header
-                cursor -= EntryHeader::serialized_len();
-                let header = EntryHeader {
-                    key_len: key_len as u32,
-                    value_len: value_len as u32,
-                    sequence,
-                    compression,
-                    checksum,
-                };
-                header.write(&mut self.buffer[cursor..cursor + EntryHeader::serialized_len()]);
+                std::io::copy(&mut &view[..], &mut self.buffer).map_err(DeviceError::from)?;
             }
             Compression::Zstd => {
-                // reserve space for header
-                let mut cursor = self.buffer.len() + EntryHeader::serialized_len();
-                unsafe { self.buffer.set_len(cursor) };
-
-                // write compressed value
                 zstd::stream::copy_encode(&mut &view[..], &mut self.buffer, 0)
                     .map_err(DeviceError::from)?;
-                let value_len = self.buffer.len() - cursor;
-
-                // write key
-                cursor += value_len;
-                self.buffer.reserve_exact(key_len);
-                unsafe { self.buffer.set_len(cursor + key_len) };
-                key.write(&mut self.buffer[cursor..cursor + key_len]);
-
-                // calculate checksum
-                cursor -= value_len;
-                let checksum = checksum(&self.buffer[cursor..cursor + value_len + key_len]);
-
-                // write entry header
-                cursor -= EntryHeader::serialized_len();
-                let header = EntryHeader {
-                    key_len: key_len as u32,
-                    value_len: value_len as u32,
-                    sequence,
-                    compression,
-                    checksum,
-                };
-                header.write(&mut self.buffer[cursor..cursor + EntryHeader::serialized_len()]);
             }
         }
+        let compressed_value_len = self.buffer.len() - cursor;
+
+        // write key
+        cursor += compressed_value_len;
+        self.buffer.reserve_exact(key.serialized_len());
+        unsafe { self.buffer.set_len(cursor + key.serialized_len()) };
+        key.write(&mut self.buffer[cursor..cursor + key.serialized_len()]);
+
+        // calculate checksum
+        cursor -= compressed_value_len;
+        let checksum =
+            checksum(&self.buffer[cursor..cursor + compressed_value_len + key.serialized_len()]);
+
+        // write entry header
+        cursor -= EntryHeader::serialized_len();
+        let header = EntryHeader {
+            key_len: key.serialized_len() as u32,
+            value_len: compressed_value_len as u32,
+            sequence,
+            compression,
+            checksum,
+        };
+        header.write(&mut self.buffer[cursor..cursor + EntryHeader::serialized_len()]);
 
         // 2. if size exceeds region limit, rollback write and return
         if self.offset + self.buffer.len() > self.device.region_size() {
@@ -289,8 +252,6 @@ where
             return Err(BufferError::NotEnough {
                 entry: Entry {
                     key,
-                    key_len,
-                    value_len,
                     sequence,
                     view,
                 },
@@ -305,8 +266,6 @@ where
         self.entries.push(PositionedEntry {
             entry: Entry {
                 key,
-                key_len,
-                value_len,
                 sequence,
                 view,
             },
@@ -342,12 +301,9 @@ mod tests {
 
     fn ent(view: RingBufferView) -> Entry {
         let key = Arc::new(());
-        let value_len = view.len();
         Entry {
             key,
             view,
-            key_len: 0,
-            value_len,
             sequence: 0,
         }
     }
diff --git a/foyer-storage/src/compress.rs b/foyer-storage/src/compress.rs
index 268bf425..37607445 100644
--- a/foyer-storage/src/compress.rs
+++ b/foyer-storage/src/compress.rs
@@ -14,7 +14,7 @@
 
 // TODO(MrCroxx): unify compress interface?
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Compression {
     None,
     Zstd,
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index e8f85d04..631b8fb5 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -35,8 +35,6 @@ pub struct Entry {
     ///
     /// Use `dyn Any` here to avoid contagious generic type.
     pub key: Arc<dyn Any + Send + Sync>,
-    pub key_len: usize,
-    pub value_len: usize,
     pub sequence: Sequence,
 
     /// Hold a view of referenced buffer, for lookup and prevent from releasing.
@@ -46,8 +44,6 @@ pub struct Entry {
 impl Debug for Entry {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Entry")
-            .field("key_len", &self.key_len)
-            .field("value_len", &self.value_len)
             .field("sequence", &self.sequence)
             .field("view", &self.view)
             .finish()
@@ -59,8 +55,6 @@ impl Clone for Entry {
         Self {
             key: Arc::clone(&self.key),
             view: self.view.clone(),
-            key_len: self.key_len,
-            value_len: self.value_len,
             sequence: self.sequence,
         }
     }
@@ -200,8 +194,6 @@ where
                 Entry {
                     key,
                     view: _,
-                    key_len,
-                    value_len,
                     sequence,
                 },
             region,
@@ -212,12 +204,10 @@ where
             bytes += len;
             let key = key.downcast::<K>().unwrap();
             let index = Index::Region {
-                view: self.region_manager.region(&region).view(
-                    offset as u32,
-                    len as u32,
-                    key_len as u32,
-                    value_len as u32,
-                ),
+                view: self
+                    .region_manager
+                    .region(&region)
+                    .view(offset as u32, len as u32),
             };
             let item = Item::new(sequence, index);
             self.catalog.insert(key, item);
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index d1aa8cea..a1343f8b 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -624,8 +624,6 @@ where
         let flusher = sequence as usize % self.inner.flusher_entry_txs.len();
         self.inner.flusher_entry_txs[flusher]
             .send(Entry {
-                key_len: key.serialized_len(),
-                value_len: value.serialized_len(),
                 sequence,
                 key,
                 view,
@@ -987,12 +985,7 @@ where
         let info = Item::new(
             header.sequence,
             Index::Region {
-                view: self.region.view(
-                    self.cursor as u32,
-                    entry_len as u32,
-                    header.key_len,
-                    header.value_len,
-                ),
+                view: self.region.view(self.cursor as u32, entry_len as u32),
             },
         );
 
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 1ad06e7c..af50b10d 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -91,14 +91,12 @@ where
         }
     }
 
-    pub fn view(&self, offset: u32, len: u32, key_len: u32, value_len: u32) -> RegionView {
+    pub fn view(&self, offset: u32, len: u32) -> RegionView {
         self.refs.fetch_add(1, Ordering::SeqCst);
         RegionView {
             id: self.id,
             offset,
             len,
-            key_len,
-            value_len,
             refs: Arc::clone(&self.refs),
         }
     }
@@ -207,8 +205,6 @@ pub struct RegionView {
     id: RegionId,
     offset: u32,
     len: u32,
-    key_len: u32,
-    value_len: u32,
     refs: Arc<AtomicUsize>,
 }
 
@@ -219,8 +215,6 @@ impl Clone for RegionView {
             id: self.id,
             offset: self.offset,
             len: self.len,
-            key_len: self.key_len,
-            value_len: self.value_len,
             refs: Arc::clone(&self.refs),
         }
     }
@@ -245,14 +239,6 @@ impl RegionView {
         &self.len
     }
 
-    pub fn key_len(&self) -> &u32 {
-        &self.key_len
-    }
-
-    pub fn value_len(&self) -> &u32 {
-        &self.value_len
-    }
-
     pub fn refs(&self) -> &Arc<AtomicUsize> {
         &self.refs
     }

From 7b0cc9fa229f38e544edd3a5e21c39204e032c31 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 3 Nov 2023 19:07:59 +0800
Subject: [PATCH 165/261] feat: add lz4 support (#202)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs     |  7 ++-----
 foyer-storage/Cargo.toml            |  1 +
 foyer-storage/src/buffer.rs         |  9 +++++++++
 foyer-storage/src/compress.rs       | 20 +++++++++++++++++++
 foyer-storage/src/generic.rs        | 23 +++++++++++++++++++++-
 foyer-storage/tests/storage_test.rs | 30 +++++++++++++++++++++++++++++
 6 files changed, 84 insertions(+), 6 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 0708f87c..3352eb5b 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -540,11 +540,8 @@ async fn main() {
         args.clean_region_threshold
     };
 
-    let compression = match args.compression.as_str() {
-        "none" => Compression::None,
-        "zstd" => Compression::Zstd,
-        _ => panic!("unsupported compression algorithm"),
-    };
+    let compression = Compression::try_from_str(args.compression.as_str())
+        .expect("unsupported compression algorithm");
 
     let config = LfuFsStoreConfig {
         name: "".to_string(),
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 2cb90693..6f602bcf 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -22,6 +22,7 @@ foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.11"
 libc = "0.2"
+lz4 = "1.24"
 memoffset = "0.9"
 nix = { version = "0.27", features = ["fs", "mman", "uio"] }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index fb2b53f8..12a60f66 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -221,6 +221,15 @@ where
                 zstd::stream::copy_encode(&mut &view[..], &mut self.buffer, 0)
                     .map_err(DeviceError::from)?;
             }
+            Compression::Lz4 => {
+                let mut encoder = lz4::EncoderBuilder::new()
+                    .checksum(lz4::ContentChecksum::NoChecksum)
+                    .build(&mut self.buffer)
+                    .map_err(DeviceError::from)?;
+                std::io::copy(&mut &view[..], &mut encoder).map_err(DeviceError::from)?;
+                let (_w, res) = encoder.finish();
+                res.map_err(DeviceError::from)?;
+            }
         }
         let compressed_value_len = self.buffer.len() - cursor;
 
diff --git a/foyer-storage/src/compress.rs b/foyer-storage/src/compress.rs
index 37607445..32d5202d 100644
--- a/foyer-storage/src/compress.rs
+++ b/foyer-storage/src/compress.rs
@@ -18,6 +18,7 @@
 pub enum Compression {
     None,
     Zstd,
+    Lz4,
 }
 
 impl Compression {
@@ -25,6 +26,7 @@ impl Compression {
         match self {
             Self::None => 0,
             Self::Zstd => 1,
+            Self::Lz4 => 2,
         }
     }
 
@@ -32,6 +34,24 @@ impl Compression {
         match v {
             0 => Some(Self::None),
             1 => Some(Self::Zstd),
+            2 => Some(Self::Lz4),
+            _ => None,
+        }
+    }
+
+    pub fn to_str(&self) -> &str {
+        match self {
+            Self::None => "none",
+            Self::Zstd => "zstd",
+            Self::Lz4 => "lz4",
+        }
+    }
+
+    pub fn try_from_str(s: &str) -> Option<Self> {
+        match s {
+            "none" => Some(Self::None),
+            "zstd" => Some(Self::Zstd),
+            "lz4" => Some(Self::Lz4),
             _ => None,
         }
     }
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index a1343f8b..e1514c1c 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -852,7 +852,28 @@ where
         Compression::Zstd => {
             let mut decompressed =
                 Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
-            if let Err(e) = zstd::stream::copy_decode(&mut &compressed[..], &mut decompressed) {
+            if let Err(e) = zstd::stream::copy_decode(compressed, &mut decompressed) {
+                tracing::warn!("decompress error: {}", e);
+                return None;
+            }
+            V::read(&decompressed[..])
+        }
+        Compression::Lz4 => {
+            let mut decompressed =
+                Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
+            let mut decoder = match lz4::Decoder::new(compressed) {
+                Ok(decoder) => decoder,
+                Err(e) => {
+                    tracing::warn!("decompress error: {}", e);
+                    return None;
+                }
+            };
+            if let Err(e) = std::io::copy(&mut decoder, &mut decompressed) {
+                tracing::warn!("decompress error: {}", e);
+                return None;
+            }
+            let (_r, res) = decoder.finish();
+            if let Err(e) = res {
                 tracing::warn!("decompress error: {}", e);
                 return None;
             }
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 3887e261..9e12073b 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -175,6 +175,36 @@ async fn test_store_zstd() {
     test_storage::<Store<_, _>>(config.into(), recorder).await;
 }
 
+#[tokio::test]
+async fn test_store_lz4() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let recorder = Arc::new(JudgeRecorder::default());
+    let config = FifoFsStoreConfig {
+        name: "".to_string(),
+        eviction_config: FifoConfig,
+        device_config: FsDeviceConfig {
+            dir: PathBuf::from(tempdir.path()),
+            capacity: 4 * MB,
+            file_capacity: 1 * MB,
+            align: 4 * KB,
+            io_size: 4 * KB,
+        },
+        ring_buffer_capacity: 16 * MB,
+        catalog_bits: 1,
+        admissions: vec![recorder.clone()],
+        reinsertions: vec![recorder.clone()],
+        flusher_buffer_size: 0,
+        flushers: 1,
+        reclaimers: 1,
+        reclaim_rate_limit: 0,
+        clean_region_threshold: 1,
+        recover_concurrency: 2,
+        compression: Compression::Lz4,
+    };
+
+    test_storage::<Store<_, _>>(config.into(), recorder).await;
+}
+
 #[tokio::test]
 async fn test_lazy_store() {
     let tempdir = tempfile::tempdir().unwrap();

From 438344b406e2c12d1da10394bc127a1e9fb3af73 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 3 Nov 2023 19:42:48 +0800
Subject: [PATCH 166/261] chore: tiny refactor about compression (#203)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/src/main.rs |  6 ++-
 foyer-storage/src/compress.rs   | 70 ++++++++++++++++++++++++++-------
 foyer-storage/src/generic.rs    |  2 +-
 3 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 3352eb5b..788e3017 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -41,7 +41,6 @@ use foyer_storage::{
         rated_random::RatedRandomAdmissionPolicy, rated_ticket::RatedTicketAdmissionPolicy,
         AdmissionPolicy,
     },
-    compress::Compression,
     device::fs::FsDeviceConfig,
     error::Result,
     reinsertion::{
@@ -540,7 +539,10 @@ async fn main() {
         args.clean_region_threshold
     };
 
-    let compression = Compression::try_from_str(args.compression.as_str())
+    let compression = args
+        .compression
+        .as_str()
+        .try_into()
         .expect("unsupported compression algorithm");
 
     let config = LfuFsStoreConfig {
diff --git a/foyer-storage/src/compress.rs b/foyer-storage/src/compress.rs
index 32d5202d..53cada0f 100644
--- a/foyer-storage/src/compress.rs
+++ b/foyer-storage/src/compress.rs
@@ -14,6 +14,10 @@
 
 // TODO(MrCroxx): unify compress interface?
 
+use anyhow::anyhow;
+
+const NOT_SUPPORT: &str = "compression algorithm not support";
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Compression {
     None,
@@ -30,15 +34,6 @@ impl Compression {
         }
     }
 
-    pub fn try_from_u8(v: u8) -> Option<Self> {
-        match v {
-            0 => Some(Self::None),
-            1 => Some(Self::Zstd),
-            2 => Some(Self::Lz4),
-            _ => None,
-        }
-    }
-
     pub fn to_str(&self) -> &str {
         match self {
             Self::None => "none",
@@ -46,13 +41,58 @@ impl Compression {
             Self::Lz4 => "lz4",
         }
     }
+}
+
+impl From<Compression> for u8 {
+    fn from(value: Compression) -> Self {
+        match value {
+            Compression::None => 0,
+            Compression::Zstd => 1,
+            Compression::Lz4 => 2,
+        }
+    }
+}
+
+impl From<Compression> for &str {
+    fn from(value: Compression) -> Self {
+        match value {
+            Compression::None => "none",
+            Compression::Zstd => "zstd",
+            Compression::Lz4 => "lz4",
+        }
+    }
+}
+
+impl TryFrom<u8> for Compression {
+    type Error = anyhow::Error;
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            0 => Ok(Self::None),
+            1 => Ok(Self::Zstd),
+            2 => Ok(Self::Lz4),
+            _ => Err(anyhow!(NOT_SUPPORT)),
+        }
+    }
+}
+
+impl TryFrom<&str> for Compression {
+    type Error = anyhow::Error;
 
-    pub fn try_from_str(s: &str) -> Option<Self> {
-        match s {
-            "none" => Some(Self::None),
-            "zstd" => Some(Self::Zstd),
-            "lz4" => Some(Self::Lz4),
-            _ => None,
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        match value {
+            "none" => Ok(Self::None),
+            "zstd" => Ok(Self::Zstd),
+            "lz4" => Ok(Self::Lz4),
+            _ => Err(anyhow!(NOT_SUPPORT)),
         }
     }
 }
+
+impl TryFrom<String> for Compression {
+    type Error = anyhow::Error;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        Self::try_from(value.as_str())
+    }
+}
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index e1514c1c..a6eda6aa 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -818,7 +818,7 @@ impl EntryHeader {
         if magic != ENTRY_MAGIC {
             return None;
         }
-        let compression = Compression::try_from_u8(v as u8)?;
+        let compression = Compression::try_from(v as u8).ok()?;
 
         Some(Self {
             key_len,

From f9b7d1bdb3603758ad2d23fa5f6861e932b1121a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 6 Nov 2023 15:45:42 +0800
Subject: [PATCH 167/261] feat: bench tool gen data based on real text (#204)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/etc/sample.txt | 676 +++++++++++++++++++++++++++++
 foyer-storage-bench/src/analyze.rs |   9 +-
 foyer-storage-bench/src/main.rs    |   7 +-
 foyer-storage-bench/src/text.rs    |  28 ++
 4 files changed, 716 insertions(+), 4 deletions(-)
 create mode 100644 foyer-storage-bench/etc/sample.txt
 create mode 100644 foyer-storage-bench/src/text.rs

diff --git a/foyer-storage-bench/etc/sample.txt b/foyer-storage-bench/etc/sample.txt
new file mode 100644
index 00000000..ab729b88
--- /dev/null
+++ b/foyer-storage-bench/etc/sample.txt
@@ -0,0 +1,676 @@
+!!! This is just a sample text, not a real license. !!!
+
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.
diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index d4d09d52..e518403b 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -318,13 +318,17 @@ pub fn analyze(
 pub async fn monitor(
     iostat_path: impl AsRef<Path>,
     interval: Duration,
+    total_secs: u64,
     metrics: Metrics,
     mut stop: broadcast::Receiver<()>,
 ) {
     let mut stat = iostat(&iostat_path);
     let mut metrics_dump = metrics.dump();
+
+    let start = Instant::now();
+
     loop {
-        let start = Instant::now();
+        let now = Instant::now();
         match stop.try_recv() {
             Err(broadcast::error::TryRecvError::Empty) => {}
             _ => return,
@@ -335,12 +339,13 @@ pub async fn monitor(
         let new_metrics_dump = metrics.dump();
         let analysis = analyze(
             // interval may have ~ +7% error
-            start.elapsed(),
+            now.elapsed(),
             &stat,
             &new_stat,
             &metrics_dump,
             &new_metrics_dump,
         );
+        println!("[{}s/{}s]", start.elapsed().as_secs(), total_secs);
         println!("{}", analysis);
         stat = new_stat;
         metrics_dump = new_metrics_dump;
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 788e3017..5e9fb38d 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -18,6 +18,7 @@
 mod analyze;
 mod export;
 mod rate;
+mod text;
 mod utils;
 
 use std::{
@@ -60,6 +61,7 @@ use rand::{
 
 use export::MetricsExporter;
 use rate::RateLimiter;
+use text::text;
 use tokio::sync::broadcast;
 use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
 
@@ -591,6 +593,7 @@ async fn main() {
         monitor(
             iostat_path,
             Duration::from_secs(args.report_interval),
+            args.time,
             metrics,
             stop_tx.subscribe(),
         )
@@ -704,7 +707,7 @@ async fn write(
         let idx = index.fetch_add(1, Ordering::Relaxed);
         // TODO(MrCroxx): Use random content?
         let entry_size = OsRng.gen_range(entry_size_range.clone());
-        let data = vec![idx as u8; entry_size];
+        let data = text(idx as usize, entry_size);
         if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
             tokio::time::sleep(wait).await;
         }
@@ -758,7 +761,7 @@ async fn read(
 
         if let Some(buf) = res {
             let entry_size = buf.len();
-            assert_eq!(vec![idx as u8; entry_size], buf);
+            assert_eq!(text(idx as usize, entry_size), buf);
             if let Err(e) = metrics.get_hit_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);
             }
diff --git a/foyer-storage-bench/src/text.rs b/foyer-storage-bench/src/text.rs
new file mode 100644
index 00000000..dceae35d
--- /dev/null
+++ b/foyer-storage-bench/src/text.rs
@@ -0,0 +1,28 @@
+//  Copyright 2023 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+const TEXT: &[u8] = include_bytes!("../etc/sample.txt");
+
+pub fn text(offset: usize, len: usize) -> Vec<u8> {
+    let mut res = Vec::with_capacity(len);
+    let mut cursor = offset % TEXT.len();
+    let mut remain = len;
+    while remain > 0 {
+        let bytes = std::cmp::min(remain, TEXT.len() - cursor);
+        res.extend(&TEXT[cursor..cursor + bytes]);
+        cursor = (cursor + bytes) % TEXT.len();
+        remain -= bytes;
+    }
+    res
+}

From 991032a9b46da8d84ec811fd60e53f21f7045d5e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 7 Nov 2023 21:48:27 +0800
Subject: [PATCH 168/261] feat: support per entry compression algorithm (#205)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .gitignore                      |  4 +++-
 foyer-storage-bench/src/main.rs | 15 +++++++++++++++
 foyer-storage/src/buffer.rs     | 30 +++++++++++++-----------------
 foyer-storage/src/flusher.rs    | 18 +++++-------------
 foyer-storage/src/generic.rs    | 24 +++++++++++++++++++++++-
 foyer-storage/src/lazy.rs       | 15 +++++++++++++++
 foyer-storage/src/runtime.rs    |  9 +++++++++
 foyer-storage/src/storage.rs    |  6 +++++-
 foyer-storage/src/store.rs      | 25 +++++++++++++++++++++++++
 9 files changed, 113 insertions(+), 33 deletions(-)

diff --git a/.gitignore b/.gitignore
index ac819a7b..7e1c8dde 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,6 @@ Cargo.lock
 lcov.info
 
 docker-compose.override.yaml
-.tmp
\ No newline at end of file
+.tmp
+
+perf.data*
\ No newline at end of file
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 5e9fb38d..36f57991 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -42,6 +42,7 @@ use foyer_storage::{
         rated_random::RatedRandomAdmissionPolicy, rated_ticket::RatedTicketAdmissionPolicy,
         AdmissionPolicy,
     },
+    compress::Compression,
     device::fs::FsDeviceConfig,
     error::Result,
     reinsertion::{
@@ -281,6 +282,20 @@ where
             BenchStoreWriter::RuntimeStoreWriter { writer } => writer.finish(value).await,
         }
     }
+
+    fn compression(&self) -> Compression {
+        match self {
+            BenchStoreWriter::StoreWriter { writer } => writer.compression(),
+            BenchStoreWriter::RuntimeStoreWriter { writer } => writer.compression(),
+        }
+    }
+
+    fn set_compression(&mut self, compression: Compression) {
+        match self {
+            BenchStoreWriter::StoreWriter { writer } => writer.set_compression(compression),
+            BenchStoreWriter::RuntimeStoreWriter { writer } => writer.set_compression(compression),
+        }
+    }
 }
 
 #[derive(Debug)]
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 12a60f66..8ab8f870 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -168,15 +168,16 @@ where
         Entry {
             key,
             sequence,
+            compression,
             view,
         }: Entry,
-        compression: Compression,
     ) -> BufferResult<Vec<PositionedEntry>> {
         if self.region.is_none() {
             return Err(BufferError::NotEnough {
                 entry: Entry {
                     key,
                     sequence,
+                    compression,
                     view,
                 },
             });
@@ -202,6 +203,7 @@ where
                 entry: Entry {
                     key,
                     sequence,
+                    compression,
                     view,
                 },
             });
@@ -262,6 +264,7 @@ where
                 entry: Entry {
                     key,
                     sequence,
+                    compression,
                     view,
                 },
             });
@@ -276,6 +279,7 @@ where
             entry: Entry {
                 key,
                 sequence,
+                compression,
                 view,
             },
             region: self.region.unwrap(),
@@ -313,6 +317,7 @@ mod tests {
         Entry {
             key,
             view,
+            compression: Compression::None,
             sequence: 0,
         }
     }
@@ -348,7 +353,7 @@ mod tests {
             let entry = ent(view);
             assert_eq!(ring.continuum(), 0);
 
-            let res = buffer.write::<()>(entry, Compression::None).await;
+            let res = buffer.write::<()>(entry).await;
             let entry = match res {
                 Err(BufferError::NotEnough { entry }) => entry,
                 _ => panic!("should be not enough error"),
@@ -358,25 +363,16 @@ mod tests {
             assert!(entries.is_empty());
 
             // 4 ~ 12 KiB
-            let entries = buffer
-                .write::<()>(entry.clone(), Compression::None)
-                .await
-                .unwrap();
+            let entries = buffer.write::<()>(entry.clone()).await.unwrap();
             assert!(entries.is_empty());
             // 12 ~ 20 KiB
-            let entries = buffer
-                .write::<()>(entry.clone(), Compression::None)
-                .await
-                .unwrap();
+            let entries = buffer.write::<()>(entry.clone()).await.unwrap();
             assert_eq!(entries.len(), 2);
             assert_eq!(entries[0].offset, 4 * 1024);
             assert_eq!(entries[1].offset, 12 * 1024);
 
             // 20 ~ 28 KiB
-            let entries = buffer
-                .write::<()>(entry.clone(), Compression::None)
-                .await
-                .unwrap();
+            let entries = buffer.write::<()>(entry.clone()).await.unwrap();
             assert!(entries.is_empty());
             let entries = buffer.flush().await.unwrap();
             assert_eq!(entries.len(), 1);
@@ -413,7 +409,7 @@ mod tests {
             };
             let entry = ent(view);
 
-            let res = buffer.write::<()>(entry, Compression::None).await;
+            let res = buffer.write::<()>(entry).await;
             let entry = match res {
                 Err(BufferError::NotEnough { entry }) => entry,
                 _ => panic!("should be not enough error"),
@@ -423,7 +419,7 @@ mod tests {
             assert!(entries.is_empty());
 
             // 4 ~ 60 KiB
-            let entries = buffer.write::<()>(entry, Compression::None).await.unwrap();
+            let entries = buffer.write::<()>(entry).await.unwrap();
             assert_eq!(entries.len(), 1);
             assert_eq!(entries[0].offset, 4 * 1024);
 
@@ -436,7 +432,7 @@ mod tests {
             let entry = ent(view);
 
             // 60 ~ 64 KiB
-            let entries = buffer.write::<()>(entry, Compression::None).await.unwrap();
+            let entries = buffer.write::<()>(entry).await.unwrap();
             assert_eq!(entries.len(), 1);
             assert_eq!(entries[0].offset, 60 * 1024);
 
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 631b8fb5..4ea2e4ab 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -36,6 +36,7 @@ pub struct Entry {
     /// Use `dyn Any` here to avoid contagious generic type.
     pub key: Arc<dyn Any + Send + Sync>,
     pub sequence: Sequence,
+    pub compression: Compression,
 
     /// Hold a view of referenced buffer, for lookup and prevent from releasing.
     pub view: RingBufferView,
@@ -56,6 +57,7 @@ impl Clone for Entry {
             key: Arc::clone(&self.key),
             view: self.view.clone(),
             sequence: self.sequence,
+            compression: self.compression,
         }
     }
 }
@@ -68,8 +70,6 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    compression: Compression,
-
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
     catalog: Arc<Catalog<K>>,
@@ -90,10 +90,8 @@ where
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
-    #[expect(clippy::too_many_arguments)]
     pub fn new(
         default_buffer_capacity: usize,
-        compression: Compression,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         catalog: Arc<Catalog<K>>,
         device: D,
@@ -103,7 +101,6 @@ where
     ) -> Self {
         let buffer = FlushBuffer::new(device.clone(), default_buffer_capacity);
         Self {
-            compression,
             region_manager,
             catalog,
             buffer,
@@ -139,7 +136,7 @@ where
 
         let old_region = self.buffer.region();
 
-        let entry = match self.buffer.write::<K>(entry, self.compression).await {
+        let entry = match self.buffer.write::<K>(entry).await {
             Err(BufferError::NotEnough { entry }) => entry,
 
             Ok(entries) => return self.update_catalog(entries).await,
@@ -176,7 +173,7 @@ where
         );
 
         // 3. retry write
-        let entries = self.buffer.write::<K>(entry, self.compression).await?;
+        let entries = self.buffer.write::<K>(entry).await?;
         self.update_catalog(entries).await?;
 
         drop(timer);
@@ -190,12 +187,7 @@ where
 
         let timer = self.metrics.inner_op_duration_update_catalog.start_timer();
         for PositionedEntry {
-            entry:
-                Entry {
-                    key,
-                    view: _,
-                    sequence,
-                },
+            entry: Entry { key, sequence, .. },
             region,
             offset,
             len,
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index a6eda6aa..6e267322 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -220,6 +220,8 @@ where
 
     metrics: Arc<Metrics>,
 
+    compression: Compression,
+
     _marker: PhantomData<V>,
 }
 
@@ -288,6 +290,7 @@ where
             flushers_stop_tx,
             reclaimers_stop_tx,
             metrics: metrics.clone(),
+            compression: config.compression,
             _marker: PhantomData,
         };
         let store = Self {
@@ -312,7 +315,6 @@ where
             .map(|(stop_rx, entry_rx)| {
                 Flusher::new(
                     config.flusher_buffer_size,
-                    config.compression,
                     region_manager.clone(),
                     catalog.clone(),
                     device.clone(),
@@ -627,6 +629,7 @@ where
                 sequence,
                 key,
                 view,
+                compression: writer.compression,
             })
             .unwrap();
 
@@ -662,6 +665,7 @@ where
 
     is_inserted: bool,
     is_skippable: bool,
+    compression: Compression,
 }
 
 impl<K, V, D, EP, EL> GenericStoreWriter<K, V, D, EP, EL>
@@ -674,6 +678,7 @@ where
 {
     fn new(store: GenericStore<K, V, D, EP, EL>, key: K, weight: usize) -> Self {
         let judges = Judges::new(store.inner.admissions.len());
+        let compression = store.inner.compression;
         Self {
             store,
             key,
@@ -684,6 +689,7 @@ where
             duration: Duration::from_nanos(0),
             is_inserted: false,
             is_skippable: false,
+            compression,
         }
     }
 
@@ -718,6 +724,14 @@ where
     pub fn set_sequence(&mut self, sequence: Sequence) {
         self.sequence = Some(sequence);
     }
+
+    pub fn compression(&self) -> Compression {
+        self.compression
+    }
+
+    pub fn set_compression(&mut self, compression: Compression) {
+        self.compression = compression
+    }
 }
 
 impl<K, V, D, EP, EL> Debug for GenericStoreWriter<K, V, D, EP, EL>
@@ -1069,6 +1083,14 @@ where
     async fn finish(self, value: Self::Value) -> Result<bool> {
         self.finish(value).await
     }
+
+    fn compression(&self) -> Compression {
+        self.compression()
+    }
+
+    fn set_compression(&mut self, compression: Compression) {
+        self.set_compression(compression)
+    }
 }
 
 impl<K, V, D, EP, EL> Storage for GenericStore<K, V, D, EP, EL>
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 39e5a746..9ee22b4b 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -15,6 +15,7 @@
 use std::sync::{Arc, OnceLock};
 
 use crate::{
+    compress::Compression,
     error::Result,
     storage::{Storage, StorageWriter},
     store::{NoneStore, NoneStoreWriter, Store},
@@ -76,6 +77,20 @@ where
             LazyStorageWriter::None { writer } => writer.finish(value).await,
         }
     }
+
+    fn compression(&self) -> Compression {
+        match self {
+            LazyStorageWriter::Store { writer } => writer.compression(),
+            LazyStorageWriter::None { writer } => writer.compression(),
+        }
+    }
+
+    fn set_compression(&mut self, compression: Compression) {
+        match self {
+            LazyStorageWriter::Store { writer } => writer.set_compression(compression),
+            LazyStorageWriter::None { writer } => writer.set_compression(compression),
+        }
+    }
 }
 
 #[derive(Debug)]
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 78d0b4bf..2dd931cf 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -20,6 +20,7 @@ use foyer_common::{
 };
 
 use crate::{
+    compress::Compression,
     error::Result,
     lazy::LazyStore,
     storage::{Storage, StorageWriter},
@@ -99,6 +100,14 @@ where
             .await
             .unwrap()
     }
+
+    fn compression(&self) -> Compression {
+        self.writer.compression()
+    }
+
+    fn set_compression(&mut self, compression: Compression) {
+        self.writer.set_compression(compression)
+    }
 }
 
 #[derive(Debug)]
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 64eb7596..92bcdcfb 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -16,7 +16,7 @@ use std::fmt::Debug;
 
 use foyer_common::code::{Key, Value};
 
-use crate::error::Result;
+use crate::{compress::Compression, error::Result};
 use futures::Future;
 
 pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
@@ -33,6 +33,10 @@ pub trait StorageWriter: Send + Sync + Debug {
 
     fn force(&mut self);
 
+    fn compression(&self) -> Compression;
+
+    fn set_compression(&mut self, compression: Compression);
+
     fn finish(self, value: Self::Value) -> impl Future<Output = Result<bool>> + Send;
 }
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 597cd553..538cfc5f 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -22,6 +22,7 @@ use foyer_intrusive::eviction::{
 };
 
 use crate::{
+    compress::Compression,
     device::fs::FsDevice,
     error::Result,
     generic::{GenericStore, GenericStoreConfig, GenericStoreWriter},
@@ -94,6 +95,12 @@ impl<K: Key, V: Value> StorageWriter for NoneStoreWriter<K, V> {
     async fn finish(self, _: Self::Value) -> Result<bool> {
         Ok(false)
     }
+
+    fn compression(&self) -> Compression {
+        Compression::None
+    }
+
+    fn set_compression(&mut self, _: Compression) {}
 }
 
 #[derive(Debug)]
@@ -352,6 +359,24 @@ where
             StoreWriter::NoneStoreWriter { writer } => writer.finish(value).await,
         }
     }
+
+    fn compression(&self) -> Compression {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.compression(),
+            StoreWriter::LfuFsStorWriter { writer } => writer.compression(),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.compression(),
+            StoreWriter::NoneStoreWriter { writer } => writer.compression(),
+        }
+    }
+
+    fn set_compression(&mut self, compression: Compression) {
+        match self {
+            StoreWriter::LruFsStorWriter { writer } => writer.set_compression(compression),
+            StoreWriter::LfuFsStorWriter { writer } => writer.set_compression(compression),
+            StoreWriter::FifoFsStoreWriter { writer } => writer.set_compression(compression),
+            StoreWriter::NoneStoreWriter { writer } => writer.set_compression(compression),
+        }
+    }
 }
 
 impl<K, V> Storage for Store<K, V>

From bde429490b8f3abdccdb455eff9e698c381e30a4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 8 Nov 2023 15:05:16 +0800
Subject: [PATCH 169/261] feat: support coding error (#209)

coding error is always logic error, which will be reported to
caller, except when recovering.

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml      |  1 +
 foyer-common/src/code.rs     | 59 ++++++++++++++++----------
 foyer-storage/src/buffer.rs  |  4 +-
 foyer-storage/src/generic.rs | 82 +++++++++++++++++-------------------
 4 files changed, 79 insertions(+), 67 deletions(-)

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 638de565..c5d9eb7e 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -11,6 +11,7 @@ license = "Apache-2.0"
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
+anyhow = "1.0"
 bytes = "1"
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 itertools = "0.11"
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 777dae8f..3755ea3f 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -15,6 +15,9 @@
 use bytes::{Buf, BufMut};
 use paste::paste;
 
+pub type CodingError = anyhow::Error;
+pub type CodingResult<T> = Result<T, CodingError>;
+
 #[expect(unused_variables)]
 pub trait Key:
     Sized
@@ -37,11 +40,11 @@ pub trait Key:
         panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
     }
 
-    fn write(&self, buf: &mut [u8]) {
+    fn write(&self, buf: &mut [u8]) -> CodingResult<()> {
         panic!("Method `write` must be implemented for `Key` if storage is used.")
     }
 
-    fn read(buf: &[u8]) -> Self {
+    fn read(buf: &[u8]) -> CodingResult<Self> {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
 }
@@ -56,11 +59,11 @@ pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
         panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
     }
 
-    fn write(&self, buf: &mut [u8]) {
+    fn write(&self, buf: &mut [u8]) -> CodingResult<()> {
         panic!("Method `write` must be implemented for `Value` if storage is used.")
     }
 
-    fn read(buf: &[u8]) -> Self {
+    fn read(buf: &[u8]) -> CodingResult<Self> {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
 }
@@ -85,13 +88,14 @@ macro_rules! impl_key {
                     }
 
 
-                    fn write(&self, mut buf: &mut [u8]) {
-                        buf.[< put_ $type>](*self)
+                    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
+                        buf.[< put_ $type>](*self);
+                        Ok(())
                     }
 
 
-                    fn read(mut buf: &[u8]) -> Self {
-                        buf.[< get_ $type>]()
+                    fn read(mut buf: &[u8]) -> CodingResult<Self> {
+                        Ok(buf.[< get_ $type>]())
                     }
                 }
             )*
@@ -110,13 +114,14 @@ macro_rules! impl_value {
                     }
 
 
-                    fn write(&self, mut buf: &mut [u8]) {
-                        buf.[< put_ $type>](*self)
+                    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
+                        buf.[< put_ $type>](*self);
+                        Ok(())
                     }
 
 
-                    fn read(mut buf: &[u8]) -> Self {
-                        buf.[< get_ $type>]()
+                    fn read(mut buf: &[u8]) -> CodingResult<Self> {
+                        Ok(buf.[< get_ $type>]())
                     }
                 }
             )*
@@ -136,12 +141,13 @@ impl Key for Vec<u8> {
         self.len()
     }
 
-    fn write(&self, mut buf: &mut [u8]) {
+    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
         buf.put_slice(self);
+        Ok(())
     }
 
-    fn read(buf: &[u8]) -> Self {
-        buf.to_vec()
+    fn read(buf: &[u8]) -> CodingResult<Self> {
+        Ok(buf.to_vec())
     }
 }
 
@@ -154,12 +160,13 @@ impl Value for Vec<u8> {
         self.len()
     }
 
-    fn write(&self, mut buf: &mut [u8]) {
+    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
         buf.put_slice(self);
+        Ok(())
     }
 
-    fn read(buf: &[u8]) -> Self {
-        buf.to_vec()
+    fn read(buf: &[u8]) -> CodingResult<Self> {
+        Ok(buf.to_vec())
     }
 }
 
@@ -172,9 +179,13 @@ impl Key for () {
         0
     }
 
-    fn write(&self, _buf: &mut [u8]) {}
+    fn write(&self, _buf: &mut [u8]) -> CodingResult<()> {
+        Ok(())
+    }
 
-    fn read(_buf: &[u8]) -> Self {}
+    fn read(_buf: &[u8]) -> CodingResult<Self> {
+        Ok(())
+    }
 }
 
 impl Value for () {
@@ -186,7 +197,11 @@ impl Value for () {
         0
     }
 
-    fn write(&self, _buf: &mut [u8]) {}
+    fn write(&self, _buf: &mut [u8]) -> CodingResult<()> {
+        Ok(())
+    }
 
-    fn read(_buf: &[u8]) -> Self {}
+    fn read(_buf: &[u8]) -> CodingResult<Self> {
+        Ok(())
+    }
 }
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 8ab8f870..e69e8a40 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -31,6 +31,8 @@ pub enum BufferError {
     Device(#[from] DeviceError),
     #[error("")]
     NotEnough { entry: Entry },
+    #[error("other error: {0}")]
+    Other(#[from] anyhow::Error),
 }
 
 pub type BufferResult<T> = std::result::Result<T, BufferError>;
@@ -239,7 +241,7 @@ where
         cursor += compressed_value_len;
         self.buffer.reserve_exact(key.serialized_len());
         unsafe { self.buffer.set_len(cursor + key.serialized_len()) };
-        key.write(&mut self.buffer[cursor..cursor + key.serialized_len()]);
+        key.write(&mut self.buffer[cursor..cursor + key.serialized_len()])?;
 
         // calculate checksum
         cursor -= compressed_value_len;
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 6e267322..b73a300f 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -22,9 +22,10 @@ use std::{
     time::{Duration, Instant},
 };
 
+use anyhow::anyhow;
 use bitmaps::Bitmap;
 use bytes::{Buf, BufMut};
-use foyer_common::{bits, rate::RateLimiter};
+use foyer_common::{bits, code::CodingError, rate::RateLimiter};
 use foyer_intrusive::eviction::EvictionPolicy;
 use futures::future::try_join_all;
 use itertools::Itertools;
@@ -407,7 +408,7 @@ where
 
         match index {
             crate::catalog::Index::RingBuffer { view } => {
-                let value = V::read(&view);
+                let value = V::read(&view)?;
 
                 self.inner
                     .metrics
@@ -443,17 +444,17 @@ where
                 };
 
                 let res = match read_entry::<K, V>(buf.as_ref()) {
-                    Some((_key, value)) => {
+                    Ok((_key, value)) => {
                         self.inner
                             .metrics
                             .op_bytes_lookup
                             .inc_by(value.serialized_len() as u64);
                         Ok(Some(value))
                     }
-                    None => {
+                    Err(e) => {
                         // Remove index if the storage layer fails to lookup it (because of entry magic mismatch).
                         self.inner.catalog.remove(key);
-                        Ok(None)
+                        Err(e)
                     }
                 };
 
@@ -612,7 +613,8 @@ where
             .inner_bytes_ring_buffer_remains
             .set(self.inner.ring.remains() as i64);
         let mut view = self.inner.ring.allocate(len, sequence).await;
-        value.write(&mut view);
+        value.write(&mut view)?;
+
         view.shrink_to(value.serialized_len());
         let view = view.freeze();
 
@@ -821,7 +823,7 @@ impl EntryHeader {
         buf.put_u32(v);
     }
 
-    pub fn read(mut buf: &[u8]) -> Option<Self> {
+    pub fn read(mut buf: &[u8]) -> Result<Self> {
         let key_len = buf.get_u32();
         let value_len = buf.get_u32();
         let sequence = buf.get_u64();
@@ -830,11 +832,13 @@ impl EntryHeader {
         let v = buf.get_u32();
         let magic = v & ENTRY_MAGIC_MASK;
         if magic != ENTRY_MAGIC {
-            return None;
+            return Err(
+                anyhow!("magic mismatch, expected: {}, got: {}", ENTRY_MAGIC, magic).into(),
+            );
         }
-        let compression = Compression::try_from(v as u8).ok()?;
+        let compression = Compression::try_from(v as u8)?;
 
-        Some(Self {
+        Ok(Self {
             key_len,
             value_len,
             sequence,
@@ -849,7 +853,7 @@ impl EntryHeader {
 /// # Safety
 ///
 /// `buf.len()` must excatly fit entry size
-fn read_entry<K, V>(buf: &[u8]) -> Option<(K, V)>
+fn read_entry<K, V>(buf: &[u8]) -> Result<(K, V)>
 where
     K: Key,
     V: Value,
@@ -862,54 +866,39 @@ where
     let compressed = &buf[offset..offset + header.value_len as usize];
     offset += header.value_len as usize;
     let value = match header.compression {
-        Compression::None => V::read(compressed),
+        Compression::None => V::read(compressed)?,
         Compression::Zstd => {
             let mut decompressed =
                 Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
-            if let Err(e) = zstd::stream::copy_decode(compressed, &mut decompressed) {
-                tracing::warn!("decompress error: {}", e);
-                return None;
-            }
-            V::read(&decompressed[..])
+            zstd::stream::copy_decode(compressed, &mut decompressed).map_err(CodingError::from)?;
+            V::read(&decompressed[..])?
         }
         Compression::Lz4 => {
             let mut decompressed =
                 Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
-            let mut decoder = match lz4::Decoder::new(compressed) {
-                Ok(decoder) => decoder,
-                Err(e) => {
-                    tracing::warn!("decompress error: {}", e);
-                    return None;
-                }
-            };
-            if let Err(e) = std::io::copy(&mut decoder, &mut decompressed) {
-                tracing::warn!("decompress error: {}", e);
-                return None;
-            }
+            let mut decoder = lz4::Decoder::new(compressed).map_err(CodingError::from)?;
+            std::io::copy(&mut decoder, &mut decompressed).map_err(CodingError::from)?;
             let (_r, res) = decoder.finish();
-            if let Err(e) = res {
-                tracing::warn!("decompress error: {}", e);
-                return None;
-            }
-            V::read(&decompressed[..])
+            res.map_err(CodingError::from)?;
+            V::read(&decompressed[..])?
         }
     };
 
     // read key
-    let key = K::read(&buf[offset..offset + header.key_len as usize]);
+    let key = K::read(&buf[offset..offset + header.key_len as usize])?;
     offset += header.key_len as usize;
 
     let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
     if checksum != header.checksum {
-        tracing::warn!(
-            "checksum mismatch, checksum: {}, expected: {}",
-            checksum,
+        return Err(anyhow!(
+            "magic mismatch, expected: {}, got: {}",
             header.checksum,
-        );
-        return None;
+            checksum
+        )
+        .into());
     }
 
-    Some((key, value))
+    Ok((key, value))
 }
 
 pub fn checksum(buf: &[u8]) -> u64 {
@@ -975,7 +964,7 @@ where
             return Ok(None);
         };
 
-        let Some(header) = EntryHeader::read(slice.as_ref()) else {
+        let Ok(header) = EntryHeader::read(slice.as_ref()) else {
             return Ok(None);
         };
 
@@ -1001,7 +990,10 @@ where
             // header and key are in the same block, read directly from slice
             let rel_start = EntryHeader::serialized_len() + header.value_len as usize;
             let rel_end = rel_start + header.key_len as usize;
-            let key = K::read(&slice.as_ref()[rel_start..rel_end]);
+
+            let Ok(key) = K::read(&slice.as_ref()[rel_start..rel_end]) else {
+                return Ok(None);
+            };
             drop(slice);
             key
         } else {
@@ -1012,7 +1004,9 @@ where
             let rel_start = abs_start - align_start;
             let rel_end = abs_end - align_start;
 
-            let key = K::read(&s.as_ref()[rel_start..rel_end]);
+            let Ok(key) = K::read(&s.as_ref()[rel_start..rel_end]) else {
+                return Ok(None);
+            };
             drop(s);
             key
         };
@@ -1046,7 +1040,7 @@ where
         let Some(slice) = self.region.load_range(start..end).await? else {
             return Ok(None);
         };
-        let kv = read_entry::<K, V>(slice.as_ref());
+        let kv = read_entry::<K, V>(slice.as_ref()).ok();
         drop(slice);
 
         Ok(kv)

From 4c32fe156858ea74c7c0cb4cda1e896b28d860fb Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 10 Nov 2023 16:11:43 +0800
Subject: [PATCH 170/261] chore: bump opentelemetry version (#211)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/Cargo.toml  | 13 +++++++++----
 foyer-storage-bench/src/main.rs |  4 ++--
 foyer-workspace-hack/Cargo.toml |  3 ---
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 54ecf813..1018a550 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -25,9 +25,13 @@ hyper = { version = "0.14", features = ["server", "http1", "tcp"] }
 itertools = "0.11.0"
 libc = "0.2"
 nix = { version = "0.27", features = ["fs", "mman"] }
-opentelemetry = { version = "0.20", features = ["rt-tokio"], optional = true }
-opentelemetry-otlp = { version = "0.13.0", optional = true }
-opentelemetry-semantic-conventions = { version = "0.12", optional = true }
+opentelemetry = { version = "0.21", optional = true }
+opentelemetry-otlp = { version = "0.14.0", optional = true }
+opentelemetry-semantic-conventions = { version = "0.13", optional = true }
+opentelemetry_sdk = { version = "0.21", features = [
+    "rt-tokio",
+    "trace",
+], optional = true }
 parking_lot = "0.12"
 prometheus = "0.13"
 rand = "0.8.5"
@@ -40,7 +44,7 @@ tokio = { version = "1", features = [
     "signal",
 ] }
 tracing = "0.1"
-tracing-opentelemetry = { version = "0.21", optional = true }
+tracing-opentelemetry = { version = "0.22", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [features]
@@ -48,6 +52,7 @@ deadlock = ["parking_lot/deadlock_detection", "foyer-storage/deadlock"]
 tokio-console = ["console-subscriber"]
 trace = [
     "opentelemetry",
+    "opentelemetry_sdk",
     "opentelemetry-otlp",
     "tracing-opentelemetry",
     "opentelemetry-semantic-conventions",
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 36f57991..2eab22cb 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -405,7 +405,7 @@ fn init_logger() {
 
 #[cfg(feature = "trace")]
 fn init_logger() {
-    use opentelemetry::sdk::{
+    use opentelemetry_sdk::{
         trace::{BatchConfig, Config},
         Resource,
     };
@@ -428,7 +428,7 @@ fn init_logger() {
         .with_exporter(opentelemetry_otlp::new_exporter().tonic())
         .with_trace_config(trace_config)
         .with_batch_config(batch_config)
-        .install_batch(opentelemetry::runtime::Tokio)
+        .install_batch(opentelemetry_sdk::runtime::Tokio)
         .unwrap();
     let opentelemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer);
     tracing_subscriber::registry()
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index bf0c75bf..6beec3fc 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -27,9 +27,6 @@ memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
-regex = { version = "1" }
-regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
-regex-syntax = { version = "0.8" }
 tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 tracing = { version = "0.1" }
 tracing-core = { version = "0.1" }

From ce2e22292e39746a6792e74c09718855d06dd0a4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 13 Nov 2023 16:07:27 +0800
Subject: [PATCH 171/261] feat: support insert if not exists async (#212)

* feat: support insert if not exists async

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: refactor async api impl

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml                   |  2 +-
 foyer-storage/src/storage.rs | 13 +++++++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 6af6dc06..f5245f18 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,4 +24,4 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
 # cmsketch = { path = "../cmsketch-rs" }
 
 [profile.release]
-debug = 1
+debug = "full"
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 92bcdcfb..3d5712da 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -202,10 +202,19 @@ impl<S: Storage> StorageExt for S {}
 pub trait AsyncStorageExt: Storage {
     #[tracing::instrument(skip(self, value))]
     fn insert_async(&self, key: Self::Key, value: Self::Value) {
-        let weight = key.serialized_len() + value.serialized_len();
         let store = self.clone();
         tokio::spawn(async move {
-            if let Err(e) = store.writer(key, weight).finish(value).await {
+            if let Err(e) = store.insert(key, value).await {
+                tracing::warn!("async storage insert error: {}", e);
+            }
+        });
+    }
+
+    #[tracing::instrument(skip(self, value))]
+    fn insert_if_not_exists_async(&self, key: Self::Key, value: Self::Value) {
+        let store = self.clone();
+        tokio::spawn(async move {
+            if let Err(e) = store.insert_if_not_exists(key, value).await {
                 tracing::warn!("async storage insert error: {}", e);
             }
         });

From a65ed186347d18e3d0e7c6db57d5c1492d272510 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 14 Nov 2023 16:56:32 +0800
Subject: [PATCH 172/261] chore: bench tool use madsim tokio (#213)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/Cargo.toml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 1018a550..60b5fea5 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -35,14 +35,7 @@ opentelemetry_sdk = { version = "0.21", features = [
 parking_lot = "0.12"
 prometheus = "0.13"
 rand = "0.8.5"
-tokio = { version = "1", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
+tokio = { workspace = true }
 tracing = "0.1"
 tracing-opentelemetry = { version = "0.22", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }

From 162c115d4f4efb3b96d597019bd88a3ddd0ce6a2 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 24 Nov 2023 11:47:10 +0800
Subject: [PATCH 173/261] refactor: refactor admission and reinsertion policy
 with context (#217)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/admission/mod.rs            | 19 +++++++---
 foyer-storage/src/admission/rated_random.rs   | 14 +++----
 foyer-storage/src/admission/rated_ticket.rs   | 14 +++----
 foyer-storage/src/generic.rs                  | 38 +++++++++++--------
 foyer-storage/src/reclaimer.rs                | 17 ++++-----
 foyer-storage/src/reinsertion/exist.rs        | 38 ++++---------------
 foyer-storage/src/reinsertion/mod.rs          | 20 +++++++---
 foyer-storage/src/reinsertion/rated_random.rs | 14 +++----
 foyer-storage/src/reinsertion/rated_ticket.rs | 14 +++----
 foyer-storage/src/test_utils.rs               | 30 ++++-----------
 10 files changed, 94 insertions(+), 124 deletions(-)

diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index a5329686..a0a06e16 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -12,24 +12,31 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use crate::{catalog::Catalog, metrics::Metrics};
 use foyer_common::code::{Key, Value};
-
 use std::{fmt::Debug, sync::Arc};
 
-use crate::{catalog::Catalog, metrics::Metrics};
+#[derive(Debug, Clone)]
+pub struct AdmissionContext<K>
+where
+    K: Key,
+{
+    pub catalog: Arc<Catalog<K>>,
+    pub metrics: Arc<Metrics>,
+}
 
 #[expect(unused_variables)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, catalog: &Arc<Catalog<Self::Key>>) {}
+    fn init(&self, context: AdmissionContext<Self::Key>) {}
 
-    fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
+    fn judge(&self, key: &Self::Key, weight: usize) -> bool;
 
-    fn on_insert(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
+    fn on_insert(&self, key: &Self::Key, weight: usize, judge: bool);
 
-    fn on_drop(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
+    fn on_drop(&self, key: &Self::Key, weight: usize, judge: bool);
 }
 
 pub mod rated_random;
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
index 6a235ca2..dd68b433 100644
--- a/foyer-storage/src/admission/rated_random.rs
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -12,16 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Duration};
-
+use super::AdmissionPolicy;
 use foyer_common::{
     code::{Key, Value},
     rated_random::RatedRandom,
 };
-
-use crate::metrics::Metrics;
-
-use super::AdmissionPolicy;
+use std::{fmt::Debug, marker::PhantomData, time::Duration};
 
 #[derive(Debug)]
 pub struct RatedRandomAdmissionPolicy<K, V>
@@ -56,15 +52,15 @@ where
 
     type Value = V;
 
-    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
         self.inner.judge()
     }
 
-    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+    fn on_insert(&self, _key: &Self::Key, weight: usize, judge: bool) {
         self.inner.on_insert(weight, judge)
     }
 
-    fn on_drop(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+    fn on_drop(&self, _key: &Self::Key, weight: usize, judge: bool) {
         self.inner.on_drop(weight, judge)
     }
 }
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
index 958c3c6d..59ca0a7a 100644
--- a/foyer-storage/src/admission/rated_ticket.rs
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -12,16 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc};
-
+use super::AdmissionPolicy;
 use foyer_common::{
     code::{Key, Value},
     rated_ticket::RatedTicket,
 };
-
-use crate::metrics::Metrics;
-
-use super::AdmissionPolicy;
+use std::{fmt::Debug, marker::PhantomData};
 
 #[derive(Debug)]
 pub struct RatedTicketAdmissionPolicy<K, V>
@@ -56,13 +52,13 @@ where
 
     type Value = V;
 
-    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
         self.inner.probe()
     }
 
-    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {
+    fn on_insert(&self, _key: &Self::Key, weight: usize, _judge: bool) {
         self.inner.reduce(weight as f64);
     }
 
-    fn on_drop(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+    fn on_drop(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 }
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index b73a300f..6902f15e 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -12,6 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use anyhow::anyhow;
+use bitmaps::Bitmap;
+use bytes::{Buf, BufMut};
+use foyer_common::{bits, code::CodingError, rate::RateLimiter};
+use foyer_intrusive::eviction::EvictionPolicy;
+use futures::future::try_join_all;
+use itertools::Itertools;
+use parking_lot::Mutex;
 use std::{
     fmt::Debug,
     marker::PhantomData,
@@ -21,15 +29,6 @@ use std::{
     },
     time::{Duration, Instant},
 };
-
-use anyhow::anyhow;
-use bitmaps::Bitmap;
-use bytes::{Buf, BufMut};
-use foyer_common::{bits, code::CodingError, rate::RateLimiter};
-use foyer_intrusive::eviction::EvictionPolicy;
-use futures::future::try_join_all;
-use itertools::Itertools;
-use parking_lot::Mutex;
 use tokio::{
     sync::{broadcast, mpsc, Semaphore},
     task::JoinHandle,
@@ -37,7 +36,7 @@ use tokio::{
 use twox_hash::XxHash64;
 
 use crate::{
-    admission::AdmissionPolicy,
+    admission::{AdmissionContext, AdmissionPolicy},
     catalog::{Catalog, Index, Item, Sequence},
     compress::Compression,
     device::Device,
@@ -48,7 +47,7 @@ use crate::{
     reclaimer::Reclaimer,
     region::{Region, RegionHeader, RegionId, REGION_MAGIC},
     region_manager::{RegionEpItemAdapter, RegionManager},
-    reinsertion::ReinsertionPolicy,
+    reinsertion::{ReinsertionContext, ReinsertionPolicy},
     ring::RingBuffer,
     storage::{Storage, StorageWriter},
 };
@@ -298,11 +297,20 @@ where
             inner: Arc::new(inner),
         };
 
+        let admission_context = AdmissionContext {
+            catalog: catalog.clone(),
+            metrics: metrics.clone(),
+        };
+        let reinsertion_context = ReinsertionContext {
+            catalog: catalog.clone(),
+            metrics: metrics.clone(),
+        };
+
         for admission in store.inner.admissions.iter() {
-            admission.init(&store.inner.catalog);
+            admission.init(admission_context.clone());
         }
         for reinsertion in store.inner.reinsertions.iter() {
-            reinsertion.init(&store.inner.catalog);
+            reinsertion.init(reinsertion_context.clone());
         }
 
         let reclaim_rate_limiter = match config.reclaim_rate_limit {
@@ -565,7 +573,7 @@ where
 
     fn judge_inner(&self, writer: &mut GenericStoreWriter<K, V, D, EP, EL>) {
         for (index, admission) in self.inner.admissions.iter().enumerate() {
-            let judge = admission.judge(&writer.key, writer.weight, &self.inner.metrics);
+            let judge = admission.judge(&writer.key, writer.weight);
             writer.judges.set(index, judge);
         }
         writer.is_judged = true;
@@ -596,7 +604,7 @@ where
 
         for (i, admission) in self.inner.admissions.iter().enumerate() {
             let judge = writer.judges.get(i);
-            admission.on_insert(&key, writer.weight, &self.inner.metrics, judge);
+            admission.on_insert(&key, writer.weight, judge);
         }
 
         // only write value to ring buffer
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 7c65b163..b997b4ef 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -12,11 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    sync::{atomic::Ordering, Arc},
-    time::Duration,
-};
-
 use crate::{
     device::Device,
     error::Result,
@@ -32,6 +27,10 @@ use foyer_common::{
     rate::RateLimiter,
 };
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use std::{
+    sync::{atomic::Ordering, Arc},
+    time::Duration,
+};
 use tokio::sync::broadcast;
 
 #[derive(Debug)]
@@ -151,13 +150,13 @@ where
 
                     let mut judges = Judges::new(reinsertions.len());
                     for (index, reinsertion) in reinsertions.iter().enumerate() {
-                        let judge = reinsertion.judge(&key, weight, &metrics);
+                        let judge = reinsertion.judge(&key, weight);
                         judges.set(index, judge);
                     }
                     if !judges.judge() {
                         for (index, reinsertion) in reinsertions.iter().enumerate() {
                             let judge = judges.get(index);
-                            reinsertion.on_drop(&key, weight, &metrics, judge);
+                            reinsertion.on_drop(&key, weight, judge);
                         }
                         continue;
                     }
@@ -177,12 +176,12 @@ where
                     if writer.finish(value).await? {
                         for (index, reinsertion) in reinsertions.iter().enumerate() {
                             let judge = judges.get(index);
-                            reinsertion.on_insert(&key, weight, &metrics, judge);
+                            reinsertion.on_insert(&key, weight, judge);
                         }
                     } else {
                         for (index, reinsertion) in reinsertions.iter().enumerate() {
                             let judge = judges.get(index);
-                            reinsertion.on_drop(&key, weight, &metrics, judge);
+                            reinsertion.on_drop(&key, weight, judge);
                         }
                         // The writer is already been judged and admitted, but not inserted successfully and skipped.
                         // That means allocating timeouts and there is no clean region available.
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index fae2a696..eaac9ab2 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -12,17 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use super::{ReinsertionContext, ReinsertionPolicy};
+use crate::catalog::Catalog;
+use foyer_common::code::{Key, Value};
 use std::{
     marker::PhantomData,
     sync::{Arc, OnceLock},
 };
 
-use foyer_common::code::{Key, Value};
-
-use crate::catalog::Catalog;
-
-use super::ReinsertionPolicy;
-
 #[derive(Debug)]
 pub struct ExistReinsertionPolicy<K, V>
 where
@@ -55,35 +52,16 @@ where
 
     type Value = V;
 
-    fn init(&self, indices: &Arc<Catalog<Self::Key>>) {
-        self.catalog.get_or_init(|| indices.clone());
+    fn init(&self, context: ReinsertionContext<Self::Key>) {
+        self.catalog.get_or_init(|| context.catalog.clone());
     }
 
-    fn judge(
-        &self,
-        key: &Self::Key,
-        _weight: usize,
-        _metrics: &std::sync::Arc<crate::metrics::Metrics>,
-    ) -> bool {
+    fn judge(&self, key: &Self::Key, _weight: usize) -> bool {
         let indices = self.catalog.get().unwrap();
         indices.lookup(key).is_some()
     }
 
-    fn on_insert(
-        &self,
-        _key: &Self::Key,
-        _weight: usize,
-        _metrics: &Arc<crate::metrics::Metrics>,
-        _judge: bool,
-    ) {
-    }
+    fn on_insert(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 
-    fn on_drop(
-        &self,
-        _key: &Self::Key,
-        _weight: usize,
-        _metrics: &Arc<crate::metrics::Metrics>,
-        _judge: bool,
-    ) {
-    }
+    fn on_drop(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 }
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index fb383c56..a5fb3cc5 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -12,23 +12,31 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use foyer_common::code::{Key, Value};
-
 use crate::{catalog::Catalog, metrics::Metrics};
+use foyer_common::code::{Key, Value};
 use std::{fmt::Debug, sync::Arc};
 
+#[derive(Debug, Clone)]
+pub struct ReinsertionContext<K>
+where
+    K: Key,
+{
+    pub catalog: Arc<Catalog<K>>,
+    pub metrics: Arc<Metrics>,
+}
+
 #[expect(unused_variables)]
 pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, indices: &Arc<Catalog<Self::Key>>) {}
+    fn init(&self, context: ReinsertionContext<Self::Key>) {}
 
-    fn judge(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>) -> bool;
+    fn judge(&self, key: &Self::Key, weight: usize) -> bool;
 
-    fn on_insert(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
+    fn on_insert(&self, key: &Self::Key, weight: usize, judge: bool);
 
-    fn on_drop(&self, key: &Self::Key, weight: usize, metrics: &Arc<Metrics>, judge: bool);
+    fn on_drop(&self, key: &Self::Key, weight: usize, judge: bool);
 }
 
 pub mod exist;
diff --git a/foyer-storage/src/reinsertion/rated_random.rs b/foyer-storage/src/reinsertion/rated_random.rs
index ab10038c..e8f36e6c 100644
--- a/foyer-storage/src/reinsertion/rated_random.rs
+++ b/foyer-storage/src/reinsertion/rated_random.rs
@@ -12,16 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Duration};
-
+use super::ReinsertionPolicy;
 use foyer_common::{
     code::{Key, Value},
     rated_random::RatedRandom,
 };
-
-use crate::metrics::Metrics;
-
-use super::ReinsertionPolicy;
+use std::{fmt::Debug, marker::PhantomData, time::Duration};
 
 #[derive(Debug)]
 pub struct RatedRandomReinsertionPolicy<K, V>
@@ -56,15 +52,15 @@ where
 
     type Value = V;
 
-    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
         self.inner.judge()
     }
 
-    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+    fn on_insert(&self, _key: &Self::Key, weight: usize, judge: bool) {
         self.inner.on_insert(weight, judge)
     }
 
-    fn on_drop(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, judge: bool) {
+    fn on_drop(&self, _key: &Self::Key, weight: usize, judge: bool) {
         self.inner.on_drop(weight, judge)
     }
 }
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
index 281c01ba..9a5f76e8 100644
--- a/foyer-storage/src/reinsertion/rated_ticket.rs
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -12,16 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc};
-
+use super::ReinsertionPolicy;
 use foyer_common::{
     code::{Key, Value},
     rated_ticket::RatedTicket,
 };
-
-use crate::metrics::Metrics;
-
-use super::ReinsertionPolicy;
+use std::{fmt::Debug, marker::PhantomData};
 
 #[derive(Debug)]
 pub struct RatedTicketReinsertionPolicy<K, V>
@@ -56,13 +52,13 @@ where
 
     type Value = V;
 
-    fn judge(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
         self.inner.probe()
     }
 
-    fn on_insert(&self, _key: &Self::Key, weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {
+    fn on_insert(&self, _key: &Self::Key, weight: usize, _judge: bool) {
         self.inner.reduce(weight as f64);
     }
 
-    fn on_drop(&self, _key: &Self::Key, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+    fn on_drop(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 }
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index 5984c6ce..be7fe4cf 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -12,12 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.s
 
-use std::{collections::HashSet, marker::PhantomData, sync::Arc};
+use std::{collections::HashSet, marker::PhantomData};
 
 use foyer_common::code::{Key, Value};
 use parking_lot::Mutex;
 
-use crate::{admission::AdmissionPolicy, metrics::Metrics, reinsertion::ReinsertionPolicy};
+use crate::{admission::AdmissionPolicy, reinsertion::ReinsertionPolicy};
 
 #[derive(Debug, Clone)]
 pub enum Record<K: Key> {
@@ -83,14 +83,14 @@ where
 
     type Value = V;
 
-    fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+    fn judge(&self, key: &K, _weight: usize) -> bool {
         self.records.lock().push(Record::Admit(key.clone()));
         true
     }
 
-    fn on_insert(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+    fn on_insert(&self, _key: &K, _weight: usize, _judge: bool) {}
 
-    fn on_drop(&self, _key: &K, _weight: usize, _metrics: &Arc<Metrics>, _judge: bool) {}
+    fn on_drop(&self, _key: &K, _weight: usize, _judge: bool) {}
 }
 
 impl<K, V> ReinsertionPolicy for JudgeRecorder<K, V>
@@ -102,26 +102,12 @@ where
 
     type Value = V;
 
-    fn judge(&self, key: &K, _weight: usize, _metrics: &Arc<Metrics>) -> bool {
+    fn judge(&self, key: &K, _weight: usize) -> bool {
         self.records.lock().push(Record::Evict(key.clone()));
         false
     }
 
-    fn on_insert(
-        &self,
-        _key: &Self::Key,
-        _weight: usize,
-        _metrics: &Arc<crate::metrics::Metrics>,
-        _judge: bool,
-    ) {
-    }
+    fn on_insert(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 
-    fn on_drop(
-        &self,
-        _key: &Self::Key,
-        _weight: usize,
-        _metrics: &Arc<crate::metrics::Metrics>,
-        _judge: bool,
-    ) {
-    }
+    fn on_drop(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 }

From 9b6f5de1c392e60925da4f111e299e793434a18e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 24 Nov 2023 11:56:40 +0800
Subject: [PATCH 174/261] chore: rustfmt enable import group fmt (#218)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/continuum.rs                 |  3 +-
 foyer-common/src/erwlock.rs                   |  3 +-
 foyer-common/src/queue.rs                     |  6 ++--
 foyer-intrusive/src/collections/dlist.rs      |  3 +-
 .../src/collections/duplicated_hashmap.rs     |  9 ++----
 foyer-intrusive/src/collections/hashmap.rs    |  9 ++----
 foyer-intrusive/src/core/adapter.rs           |  5 +---
 foyer-intrusive/src/core/pointer.rs           |  3 +-
 foyer-intrusive/src/eviction/fifo.rs          |  5 ++--
 foyer-intrusive/src/eviction/lfu.rs           | 25 ++++++++---------
 foyer-intrusive/src/eviction/lru.rs           |  6 ++--
 foyer-intrusive/src/eviction/mod.rs           |  3 +-
 foyer-intrusive/src/eviction/sfifo.rs         |  7 ++---
 foyer-memory/src/lib.rs                       |  3 +-
 foyer-storage-bench/src/main.rs               |  4 +--
 foyer-storage/src/admission/mod.rs            |  6 ++--
 foyer-storage/src/admission/rated_random.rs   |  6 ++--
 foyer-storage/src/admission/rated_ticket.rs   |  6 ++--
 foyer-storage/src/buffer.rs                   |  3 +-
 foyer-storage/src/device/allocator.rs         |  3 +-
 foyer-storage/src/device/fs.rs                |  8 +++---
 foyer-storage/src/device/mod.rs               |  3 +-
 foyer-storage/src/flusher.rs                  | 12 ++++----
 foyer-storage/src/generic.rs                  | 28 ++++++++++---------
 foyer-storage/src/lazy.rs                     |  8 +++---
 foyer-storage/src/reclaimer.rs                | 24 ++++++++--------
 foyer-storage/src/region.rs                   |  7 +++--
 foyer-storage/src/region_manager.rs           |  1 -
 foyer-storage/src/reinsertion/exist.rs        |  8 ++++--
 foyer-storage/src/reinsertion/mod.rs          |  6 ++--
 foyer-storage/src/reinsertion/rated_random.rs |  6 ++--
 foyer-storage/src/reinsertion/rated_ticket.rs |  6 ++--
 foyer-storage/src/ring.rs                     | 16 ++++++-----
 foyer-storage/src/storage.rs                  |  2 +-
 foyer-storage/tests/storage_test.rs           |  3 +-
 rustfmt.toml                                  |  1 +
 36 files changed, 134 insertions(+), 123 deletions(-)

diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
index fe0bddd9..430c33b9 100644
--- a/foyer-common/src/continuum.rs
+++ b/foyer-common/src/continuum.rs
@@ -12,12 +12,13 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use itertools::Itertools;
 use std::{
     ops::Range,
     sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering},
 };
 
+use itertools::Itertools;
+
 macro_rules! def_continuum {
     ($( { $name:ident, $uint:ty, $atomic:ty }, )*) => {
         $(
diff --git a/foyer-common/src/erwlock.rs b/foyer-common/src/erwlock.rs
index b2acfb28..207734b7 100644
--- a/foyer-common/src/erwlock.rs
+++ b/foyer-common/src/erwlock.rs
@@ -12,10 +12,11 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::sync::Arc;
+
 use parking_lot::{
     lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard,
 };
-use std::sync::Arc;
 
 pub trait ErwLockInner {
     type R;
diff --git a/foyer-common/src/queue.rs b/foyer-common/src/queue.rs
index 4ab33e7e..03df30c0 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/queue.rs
@@ -12,8 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use parking_lot::Mutex;
 use std::{collections::VecDeque, fmt::Debug};
+
+use parking_lot::Mutex;
 use tokio::sync::{watch, Notify};
 
 #[derive(Debug)]
@@ -103,13 +104,14 @@ impl<T: Debug> AsyncQueue<T> {
 
 #[cfg(test)]
 mod tests {
-    use crate::queue::AsyncQueue;
     use std::{
         future::{poll_fn, Future},
         pin::pin,
         task::{Poll, Poll::Pending},
     };
 
+    use crate::queue::AsyncQueue;
+
     #[tokio::test]
     async fn test_basic() {
         let queue = AsyncQueue::new();
diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index e40a0695..38c35f5c 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -507,9 +507,8 @@ mod tests {
 
     use itertools::Itertools;
 
-    use crate::intrusive_adapter;
-
     use super::*;
+    use crate::intrusive_adapter;
 
     #[derive(Debug)]
     struct DListItem {
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index f70affa3..13075297 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -12,12 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, ptr::NonNull};
+use std::{fmt::Debug, hash::Hasher, marker::PhantomData, ptr::NonNull};
 
 use foyer_common::code::{Key, Value};
-use std::hash::Hasher;
 use twox_hash::XxHash64;
 
+use super::dlist::{DList, DListIter, DListIterMut, DListLink};
 use crate::{
     core::{
         adapter::{Adapter, KeyAdapter, Link},
@@ -26,8 +26,6 @@ use crate::{
     intrusive_adapter,
 };
 
-use super::dlist::{DList, DListIter, DListIterMut, DListLink};
-
 pub struct DuplicatedHashMapLink {
     slot_link: DListLink,
     group_link: DListLink,
@@ -336,9 +334,8 @@ mod tests {
 
     use itertools::Itertools;
 
-    use crate::{intrusive_adapter, key_adapter};
-
     use super::*;
+    use crate::{intrusive_adapter, key_adapter};
 
     #[derive(Debug)]
     struct DuplicatedHashMapItem {
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 0f50c4bc..c1bd07f2 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -12,12 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{marker::PhantomData, ptr::NonNull};
+use std::{hash::Hasher, marker::PhantomData, ptr::NonNull};
 
 use foyer_common::code::{Key, Value};
-use std::hash::Hasher;
 use twox_hash::XxHash64;
 
+use super::dlist::{DList, DListIter, DListIterMut, DListLink};
 use crate::{
     core::{
         adapter::{KeyAdapter, Link},
@@ -26,8 +26,6 @@ use crate::{
     intrusive_adapter,
 };
 
-use super::dlist::{DList, DListIter, DListIterMut, DListLink};
-
 #[derive(Debug, Default)]
 pub struct HashMapLink {
     dlist_link: DListLink,
@@ -353,9 +351,8 @@ mod tests {
 
     use itertools::Itertools;
 
-    use crate::{intrusive_adapter, key_adapter};
-
     use super::*;
+    use crate::{intrusive_adapter, key_adapter};
 
     #[derive(Debug)]
     struct HashMapItem {
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index db0fd0b7..07469f2b 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -327,11 +327,8 @@ macro_rules! priority_adapter {
 mod tests {
     use itertools::Itertools;
 
-    use crate::collections::dlist::*;
-
-    use crate::intrusive_adapter;
-
     use super::*;
+    use crate::{collections::dlist::*, intrusive_adapter};
 
     #[derive(Debug)]
     struct DListItem {
diff --git a/foyer-intrusive/src/core/pointer.rs b/foyer-intrusive/src/core/pointer.rs
index 01f7c079..ef0cfc74 100644
--- a/foyer-intrusive/src/core/pointer.rs
+++ b/foyer-intrusive/src/core/pointer.rs
@@ -240,9 +240,10 @@ unsafe impl<T: ?Sized + Debug> DowngradablePointerOps for Arc<T> {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use std::{boxed::Box, fmt::Debug, mem, pin::Pin, rc::Rc, sync::Arc};
 
+    use super::*;
+
     /// Clones a `Pointer` from a `*const Pointer::Value`
     ///
     /// This method is only safe to call if the raw pointer is known to be
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index c035fb9d..dc4f60ba 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -28,6 +28,7 @@
 
 use std::{mem::ManuallyDrop, ptr::NonNull};
 
+use super::EvictionPolicy;
 use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
@@ -37,8 +38,6 @@ use crate::{
     intrusive_adapter,
 };
 
-use super::EvictionPolicy;
-
 #[derive(Debug, Clone)]
 pub struct FifoConfig;
 
@@ -288,10 +287,10 @@ where
 mod tests {
     use std::sync::Arc;
 
-    use crate::eviction::EvictionPolicyExt;
     use itertools::Itertools;
 
     use super::*;
+    use crate::eviction::EvictionPolicyExt;
 
     #[derive(Debug)]
     struct FifoItem {
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 6923ba00..6b1d34d8 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -26,8 +26,18 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::{
+    hash::{Hash, Hasher},
+    mem::ManuallyDrop,
+    ptr::NonNull,
+};
+
+use cmsketch::CMSketchUsize;
+use twox_hash::XxHash64;
+
+use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter},
+    collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, KeyAdapter, Link},
         pointer::Pointer,
@@ -35,16 +45,6 @@ use crate::{
     intrusive_adapter,
 };
 
-use std::{mem::ManuallyDrop, ptr::NonNull};
-
-use cmsketch::CMSketchUsize;
-use std::hash::{Hash, Hasher};
-use twox_hash::XxHash64;
-
-use crate::collections::dlist::DListLink;
-
-use super::EvictionPolicy;
-
 const MIN_CAPACITY: usize = 100;
 const ERROR_THRESHOLD: f64 = 5.0;
 const HASH_COUNT: usize = 4;
@@ -560,9 +560,8 @@ mod tests {
 
     use itertools::Itertools;
 
-    use crate::key_adapter;
-
     use super::*;
+    use crate::key_adapter;
 
     #[derive(Debug)]
     struct LfuItem {
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 2390db98..e287c9c6 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -28,6 +28,7 @@
 
 use std::{mem::ManuallyDrop, ptr::NonNull};
 
+use super::EvictionPolicy;
 use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
@@ -37,8 +38,6 @@ use crate::{
     intrusive_adapter,
 };
 
-use super::EvictionPolicy;
-
 #[derive(Clone, Debug)]
 pub struct LruConfig {
     /// Insertion point of the new entry, between 0 and 1.
@@ -415,9 +414,8 @@ mod tests {
 
     use itertools::Itertools;
 
-    use crate::key_adapter;
-
     use super::*;
+    use crate::key_adapter;
 
     #[derive(Debug)]
     struct LruItem {
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index 1e64e45d..aebc72f7 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -12,9 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::core::adapter::Adapter;
 use std::fmt::Debug;
 
+use crate::core::adapter::Adapter;
+
 pub trait Config = Send + Sync + 'static + Debug + Clone;
 
 pub trait EvictionPolicy: Send + Sync + Debug + 'static {
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index 8d574799..cd422db0 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -30,6 +30,7 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use itertools::Itertools;
 
+use super::EvictionPolicy;
 use crate::{
     collections::dlist::{DList, DListIter, DListLink},
     core::{
@@ -39,8 +40,6 @@ use crate::{
     intrusive_adapter,
 };
 
-use super::EvictionPolicy;
-
 #[derive(Debug, Clone)]
 pub struct SegmentedFifoConfig {
     /// `segment_ratios` is used to compute the ratio of each segment's size.
@@ -381,12 +380,10 @@ where
 mod tests {
     use std::sync::Arc;
 
-    use crate::eviction::EvictionPolicyExt;
     use itertools::Itertools;
 
-    use crate::priority_adapter;
-
     use super::*;
+    use crate::{eviction::EvictionPolicyExt, priority_adapter};
 
     #[derive(Debug)]
     struct SegmentedFifoItem {
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index dd1bbaca..b78bb7b1 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::Arc;
+use std::{hash::Hasher, sync::Arc};
 
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::{
@@ -22,7 +22,6 @@ use foyer_intrusive::{
     intrusive_adapter, key_adapter, priority_adapter,
 };
 use parking_lot::Mutex;
-use std::hash::Hasher;
 use twox_hash::XxHash64;
 
 pub struct CacheConfig<E>
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 2eab22cb..36415a3b 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -34,7 +34,7 @@ use std::{
 
 use analyze::{analyze, monitor, Metrics};
 use clap::Parser;
-
+use export::MetricsExporter;
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
@@ -59,8 +59,6 @@ use rand::{
     rngs::{OsRng, StdRng},
     Rng, SeedableRng,
 };
-
-use export::MetricsExporter;
 use rate::RateLimiter;
 use text::text;
 use tokio::sync::broadcast;
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index a0a06e16..3912929f 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -12,10 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{catalog::Catalog, metrics::Metrics};
-use foyer_common::code::{Key, Value};
 use std::{fmt::Debug, sync::Arc};
 
+use foyer_common::code::{Key, Value};
+
+use crate::{catalog::Catalog, metrics::Metrics};
+
 #[derive(Debug, Clone)]
 pub struct AdmissionContext<K>
 where
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
index dd68b433..32702df7 100644
--- a/foyer-storage/src/admission/rated_random.rs
+++ b/foyer-storage/src/admission/rated_random.rs
@@ -12,12 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use super::AdmissionPolicy;
+use std::{fmt::Debug, marker::PhantomData, time::Duration};
+
 use foyer_common::{
     code::{Key, Value},
     rated_random::RatedRandom,
 };
-use std::{fmt::Debug, marker::PhantomData, time::Duration};
+
+use super::AdmissionPolicy;
 
 #[derive(Debug)]
 pub struct RatedRandomAdmissionPolicy<K, V>
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
index 59ca0a7a..c81f62a8 100644
--- a/foyer-storage/src/admission/rated_ticket.rs
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -12,12 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use super::AdmissionPolicy;
+use std::{fmt::Debug, marker::PhantomData};
+
 use foyer_common::{
     code::{Key, Value},
     rated_ticket::RatedTicket,
 };
-use std::{fmt::Debug, marker::PhantomData};
+
+use super::AdmissionPolicy;
 
 #[derive(Debug)]
 pub struct RatedTicketAdmissionPolicy<K, V>
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index e69e8a40..6028d29b 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -307,13 +307,12 @@ mod tests {
     use bytes::BufMut;
     use tempfile::tempdir;
 
+    use super::*;
     use crate::{
         device::fs::{FsDevice, FsDeviceConfig},
         ring::{RingBuffer, RingBufferView},
     };
 
-    use super::*;
-
     fn ent(view: RingBufferView) -> Entry {
         let key = Arc::new(());
         Entry {
diff --git a/foyer-storage/src/device/allocator.rs b/foyer-storage/src/device/allocator.rs
index 1a974b96..cb133602 100644
--- a/foyer-storage/src/device/allocator.rs
+++ b/foyer-storage/src/device/allocator.rs
@@ -12,9 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use foyer_common::bits;
 use std::alloc::{Allocator, Global};
 
+use foyer_common::bits;
+
 #[derive(Debug, Clone, Copy)]
 pub struct AlignedAllocator {
     align: usize,
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index a4301b35..dbafab04 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -19,7 +19,9 @@ use std::{
     sync::Arc,
 };
 
-use crate::region::RegionId;
+use foyer_common::range::RangeBoundsExt;
+use futures::future::try_join_all;
+use itertools::Itertools;
 
 use super::{
     allocator::AlignedAllocator,
@@ -27,9 +29,7 @@ use super::{
     error::{DeviceError, DeviceResult},
     Device, IoBuf, IoBufMut, IoRange,
 };
-use foyer_common::range::RangeBoundsExt;
-use futures::future::try_join_all;
-use itertools::Itertools;
+use crate::region::RegionId;
 
 #[derive(Debug, Clone)]
 pub struct FsDeviceConfig {
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 45eddc75..c5f9e39f 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -18,11 +18,12 @@ pub mod fs;
 
 use std::{alloc::Allocator, fmt::Debug};
 
-use crate::region::RegionId;
 use error::DeviceResult;
 use foyer_common::range::RangeBoundsExt;
 use futures::Future;
 
+use crate::region::RegionId;
+
 pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
 pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
 pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 4ea2e4ab..4741c75e 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -12,6 +12,13 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::{any::Any, fmt::Debug, sync::Arc};
+
+use foyer_common::code::Key;
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use tokio::sync::{broadcast, mpsc};
+use tracing::Instrument;
+
 use crate::{
     buffer::{BufferError, FlushBuffer, PositionedEntry},
     catalog::{Catalog, Index, Item, Sequence},
@@ -22,11 +29,6 @@ use crate::{
     region_manager::{RegionEpItemAdapter, RegionManager},
     ring::RingBufferView,
 };
-use foyer_common::code::Key;
-use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use std::{any::Any, fmt::Debug, sync::Arc};
-use tokio::sync::{broadcast, mpsc};
-use tracing::Instrument;
 
 pub struct Entry {
     /// # Safety
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 6902f15e..e748807a 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -12,16 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use anyhow::anyhow;
-use bitmaps::Bitmap;
-use bytes::{Buf, BufMut};
-use foyer_common::{bits, code::CodingError, rate::RateLimiter};
-use foyer_intrusive::eviction::EvictionPolicy;
-use futures::future::try_join_all;
-use itertools::Itertools;
-use parking_lot::Mutex;
 use std::{
     fmt::Debug,
+    hash::Hasher,
     marker::PhantomData,
     sync::{
         atomic::{AtomicU64, Ordering},
@@ -29,6 +22,19 @@ use std::{
     },
     time::{Duration, Instant},
 };
+
+use anyhow::anyhow;
+use bitmaps::Bitmap;
+use bytes::{Buf, BufMut};
+use foyer_common::{
+    bits,
+    code::{CodingError, Key, Value},
+    rate::RateLimiter,
+};
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use futures::future::try_join_all;
+use itertools::Itertools;
+use parking_lot::Mutex;
 use tokio::{
     sync::{broadcast, mpsc, Semaphore},
     task::JoinHandle,
@@ -51,9 +57,6 @@ use crate::{
     ring::RingBuffer,
     storage::{Storage, StorageWriter},
 };
-use foyer_common::code::{Key, Value};
-use foyer_intrusive::core::adapter::Link;
-use std::hash::Hasher;
 
 const DEFAULT_BROADCAST_CAPACITY: usize = 4096;
 
@@ -1147,14 +1150,13 @@ mod tests {
 
     use foyer_intrusive::eviction::fifo::{Fifo, FifoConfig, FifoLink};
 
+    use super::*;
     use crate::{
         device::fs::{FsDevice, FsDeviceConfig},
         storage::StorageExt,
         test_utils::JudgeRecorder,
     };
 
-    use super::*;
-
     type TestStore =
         GenericStore<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 9ee22b4b..8354c9ee 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -14,14 +14,15 @@
 
 use std::sync::{Arc, OnceLock};
 
+use foyer_common::code::{Key, Value};
+use tokio::task::JoinHandle;
+
 use crate::{
     compress::Compression,
     error::Result,
     storage::{Storage, StorageWriter},
     store::{NoneStore, NoneStoreWriter, Store},
 };
-use foyer_common::code::{Key, Value};
-use tokio::task::JoinHandle;
 
 #[derive(Debug)]
 pub enum LazyStorageWriter<K, V, S>
@@ -228,14 +229,13 @@ mod tests {
 
     use foyer_intrusive::eviction::fifo::FifoConfig;
 
+    use super::*;
     use crate::{
         device::fs::FsDeviceConfig,
         storage::StorageExt,
         store::{FifoFsStoreConfig, Store},
     };
 
-    use super::*;
-
     const KB: usize = 1024;
     const MB: usize = 1024 * 1024;
 
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index b997b4ef..77b5f2b9 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -12,6 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::{
+    sync::{atomic::Ordering, Arc},
+    time::Duration,
+};
+
+use bytes::BufMut;
+use foyer_common::{
+    code::{Key, Value},
+    rate::RateLimiter,
+};
+use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
+use tokio::sync::broadcast;
+
 use crate::{
     device::Device,
     error::Result,
@@ -21,17 +34,6 @@ use crate::{
     region_manager::{RegionEpItemAdapter, RegionManager},
     storage::Storage,
 };
-use bytes::BufMut;
-use foyer_common::{
-    code::{Key, Value},
-    rate::RateLimiter,
-};
-use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
-use std::{
-    sync::{atomic::Ordering, Arc},
-    time::Duration,
-};
-use tokio::sync::broadcast;
 
 #[derive(Debug)]
 pub struct Reclaimer<K, V, D, EP, EL>
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index af50b10d..38491ab0 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -12,9 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use bytes::{Buf, BufMut};
-use foyer_common::range::RangeBoundsExt;
-use parking_lot::Mutex;
 use std::{
     collections::btree_map::{BTreeMap, Entry},
     fmt::Debug,
@@ -24,6 +21,10 @@ use std::{
         Arc,
     },
 };
+
+use bytes::{Buf, BufMut};
+use foyer_common::range::RangeBoundsExt;
+use parking_lot::Mutex;
 use tokio::sync::oneshot;
 
 use crate::{
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index dc691a4c..81167307 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -20,7 +20,6 @@ use foyer_intrusive::{
     eviction::{EvictionPolicy, EvictionPolicyExt},
     intrusive_adapter, key_adapter,
 };
-
 use parking_lot::RwLock;
 
 use crate::{
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index eaac9ab2..7af2ccbc 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -12,14 +12,16 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use super::{ReinsertionContext, ReinsertionPolicy};
-use crate::catalog::Catalog;
-use foyer_common::code::{Key, Value};
 use std::{
     marker::PhantomData,
     sync::{Arc, OnceLock},
 };
 
+use foyer_common::code::{Key, Value};
+
+use super::{ReinsertionContext, ReinsertionPolicy};
+use crate::catalog::Catalog;
+
 #[derive(Debug)]
 pub struct ExistReinsertionPolicy<K, V>
 where
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index a5fb3cc5..b4fc9be2 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -12,10 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{catalog::Catalog, metrics::Metrics};
-use foyer_common::code::{Key, Value};
 use std::{fmt::Debug, sync::Arc};
 
+use foyer_common::code::{Key, Value};
+
+use crate::{catalog::Catalog, metrics::Metrics};
+
 #[derive(Debug, Clone)]
 pub struct ReinsertionContext<K>
 where
diff --git a/foyer-storage/src/reinsertion/rated_random.rs b/foyer-storage/src/reinsertion/rated_random.rs
index e8f36e6c..5eb99386 100644
--- a/foyer-storage/src/reinsertion/rated_random.rs
+++ b/foyer-storage/src/reinsertion/rated_random.rs
@@ -12,12 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use super::ReinsertionPolicy;
+use std::{fmt::Debug, marker::PhantomData, time::Duration};
+
 use foyer_common::{
     code::{Key, Value},
     rated_random::RatedRandom,
 };
-use std::{fmt::Debug, marker::PhantomData, time::Duration};
+
+use super::ReinsertionPolicy;
 
 #[derive(Debug)]
 pub struct RatedRandomReinsertionPolicy<K, V>
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
index 9a5f76e8..74d70ba1 100644
--- a/foyer-storage/src/reinsertion/rated_ticket.rs
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -12,12 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use super::ReinsertionPolicy;
+use std::{fmt::Debug, marker::PhantomData};
+
 use foyer_common::{
     code::{Key, Value},
     rated_ticket::RatedTicket,
 };
-use std::{fmt::Debug, marker::PhantomData};
+
+use super::ReinsertionPolicy;
 
 #[derive(Debug)]
 pub struct RatedTicketReinsertionPolicy<K, V>
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
index 1d7ba647..495bb56a 100644
--- a/foyer-storage/src/ring.rs
+++ b/foyer-storage/src/ring.rs
@@ -12,13 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{
-    catalog::Sequence,
-    device::BufferAllocator,
-    metrics::{Metrics, METRICS},
-};
-use foyer_common::{bits::align_up, continuum::ContinuumUsize};
-use itertools::Itertools;
 use std::{
     alloc::Global,
     fmt::Debug,
@@ -30,6 +23,15 @@ use std::{
     time::{Duration, Instant},
 };
 
+use foyer_common::{bits::align_up, continuum::ContinuumUsize};
+use itertools::Itertools;
+
+use crate::{
+    catalog::Sequence,
+    device::BufferAllocator,
+    metrics::{Metrics, METRICS},
+};
+
 pub struct RingBuffer<A = Global>
 where
     A: BufferAllocator,
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 3d5712da..d02cb553 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -15,9 +15,9 @@
 use std::fmt::Debug;
 
 use foyer_common::code::{Key, Value};
+use futures::Future;
 
 use crate::{compress::Compression, error::Result};
-use futures::Future;
 
 pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
 
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 9e12073b..07fac38f 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -15,6 +15,8 @@
 #![feature(lint_reasons)]
 #![expect(clippy::identity_op)]
 
+use std::{path::PathBuf, sync::Arc, time::Duration};
+
 use foyer_intrusive::eviction::fifo::FifoConfig;
 use foyer_storage::{
     compress::Compression,
@@ -25,7 +27,6 @@ use foyer_storage::{
     store::{FifoFsStoreConfig, Store},
     test_utils::JudgeRecorder,
 };
-use std::{path::PathBuf, sync::Arc, time::Duration};
 
 const KB: usize = 1024;
 const MB: usize = 1024 * 1024;
diff --git a/rustfmt.toml b/rustfmt.toml
index 946ebb58..6c08e327 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1,2 +1,3 @@
 imports_granularity = "Crate"
+group_imports = "StdExternalCrate"
 tab_spaces = 4
\ No newline at end of file

From 64c47bcd8e6c8ffc67e5d3827a45adb7d0a01e5b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 24 Nov 2023 13:31:17 +0800
Subject: [PATCH 175/261] chore: upgrade hyper to 1.0 (#219)

* chore: upgrade hyper to 1.0

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix fmt

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage-bench/Cargo.toml    | 10 +++++++-
 foyer-storage-bench/src/export.rs | 39 ++++++++++++++++++++++---------
 foyer-workspace-hack/Cargo.toml   |  9 ++++---
 3 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 60b5fea5..a1f1aada 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -21,7 +21,15 @@ foyer-storage = { path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
-hyper = { version = "0.14", features = ["server", "http1", "tcp"] }
+http-body-util = "0.1"
+hyper = { version = "1.0", features = ["server", "http1", "http2"] }
+hyper-util = { version = "0.1", features = [
+    "server",
+    "server-auto",
+    "http1",
+    "http2",
+    "tokio",
+] }
 itertools = "0.11.0"
 libc = "0.2"
 nix = { version = "0.27", features = ["fs", "mman"] }
diff --git a/foyer-storage-bench/src/export.rs b/foyer-storage-bench/src/export.rs
index 7c21e071..3fcf7d10 100644
--- a/foyer-storage-bench/src/export.rs
+++ b/foyer-storage-bench/src/export.rs
@@ -15,12 +15,15 @@
 use std::net::SocketAddr;
 
 use foyer_storage::metrics::get_metrics_registry;
+use http_body_util::Full;
 use hyper::{
+    body::{Body, Bytes},
     header::CONTENT_TYPE,
-    service::{make_service_fn, service_fn},
-    Body, Error, Request, Response, Server,
+    service::service_fn,
+    Request, Response,
 };
 use prometheus::{Encoder, TextEncoder};
+use tokio::net::TcpListener;
 
 pub struct MetricsExporter;
 
@@ -28,18 +31,32 @@ impl MetricsExporter {
     pub fn init(addr: SocketAddr) {
         tokio::spawn(async move {
             tracing::info!("Prometheus service is set up on http://{}", addr);
-            if let Err(e) = Server::bind(&addr)
-                .serve(make_service_fn(|_| async move {
-                    Ok::<_, Error>(service_fn(Self::serve))
-                }))
-                .await
-            {
-                tracing::error!("Prometheus service error: {}", e);
+
+            let listener = TcpListener::bind(addr).await.unwrap();
+            loop {
+                let stream = match listener.accept().await {
+                    Ok((stream, _addr)) => stream,
+                    Err(e) => {
+                        tracing::error!("accept conn error: {}", e);
+                        continue;
+                    }
+                };
+                let io = hyper_util::rt::TokioIo::new(stream);
+                tokio::spawn(async move {
+                    if let Err(e) = hyper_util::server::conn::auto::Builder::new(
+                        hyper_util::rt::TokioExecutor::new(),
+                    )
+                    .serve_connection(io, service_fn(Self::serve))
+                    .await
+                    {
+                        tracing::error!("Prometheus service error: {}", e);
+                    }
+                });
             }
         });
     }
 
-    async fn serve(_request: Request<Body>) -> anyhow::Result<Response<Body>> {
+    async fn serve(_request: Request<impl Body + Sized>) -> anyhow::Result<Response<Full<Bytes>>> {
         let encoder = TextEncoder::new();
         let mut buffer = Vec::with_capacity(4096);
         let metrics = get_metrics_registry().gather();
@@ -47,7 +64,7 @@ impl MetricsExporter {
         let response = Response::builder()
             .status(200)
             .header(CONTENT_TYPE, encoder.format_type())
-            .body(Body::from(buffer))?;
+            .body(Full::new(Bytes::from(buffer)))?;
         Ok(response)
     }
 }
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 6beec3fc..dd0d60aa 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -20,23 +20,22 @@ futures-core = { version = "0.3" }
 futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
-hyper = { version = "0.14", features = ["full"] }
 itertools = { version = "0.11" }
 libc = { version = "0.2", features = ["extra_traits"] }
+log = { version = "0.4", default-features = false, features = ["std"] }
 memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
 tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
-tracing = { version = "0.1" }
+tokio-util = { version = "0.7", features = ["codec", "io"] }
+tower = { version = "0.4", features = ["balance", "buffer", "limit", "timeout", "util"] }
+tracing = { version = "0.1", features = ["log"] }
 tracing-core = { version = "0.1" }
 
 [build-dependencies]
 cc = { version = "1", default-features = false, features = ["parallel"] }
 either = { version = "1", default-features = false, features = ["use_std"] }
 itertools = { version = "0.11" }
-proc-macro2 = { version = "1" }
-quote = { version = "1" }
-syn = { version = "2", features = ["extra-traits", "full", "visit-mut"] }
 
 ### END HAKARI SECTION

From ca747d91f4779c0095ed0d1fe9638ed2fd17179f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 24 Nov 2023 13:42:59 +0800
Subject: [PATCH 176/261] chore: upgrade itertools dep (#220)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml         | 5 +----
 foyer-intrusive/Cargo.toml      | 2 +-
 foyer-storage-bench/Cargo.toml  | 2 +-
 foyer-storage/Cargo.toml        | 2 +-
 foyer-workspace-hack/Cargo.toml | 2 --
 5 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index c5d9eb7e..0f367928 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -14,12 +14,9 @@ normal = ["foyer-workspace-hack"]
 anyhow = "1.0"
 bytes = "1"
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
-itertools = "0.11"
+itertools = "0.12"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 rand = "0.8.5"
 tokio = { workspace = true }
 tracing = "0.1"
-
-[dev-dependencies]
-itertools = "0.11"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 412e58db..8bfe7b11 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -15,7 +15,7 @@ bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
-itertools = "0.11"
+itertools = "0.12"
 memoffset = "0.9"
 parking_lot = "0.12"
 paste = "1.0"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index a1f1aada..6c2707a7 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -30,7 +30,7 @@ hyper-util = { version = "0.1", features = [
     "http2",
     "tokio",
 ] }
-itertools = "0.11.0"
+itertools = "0.12"
 libc = "0.2"
 nix = { version = "0.27", features = ["fs", "mman"] }
 opentelemetry = { version = "0.21", optional = true }
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 6f602bcf..e1332f8d 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -20,7 +20,7 @@ foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
-itertools = "0.11"
+itertools = "0.12"
 libc = "0.2"
 lz4 = "1.24"
 memoffset = "0.9"
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index dd0d60aa..005f546b 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -20,7 +20,6 @@ futures-core = { version = "0.3" }
 futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
-itertools = { version = "0.11" }
 libc = { version = "0.2", features = ["extra_traits"] }
 log = { version = "0.4", default-features = false, features = ["std"] }
 memchr = { version = "2" }
@@ -36,6 +35,5 @@ tracing-core = { version = "0.1" }
 [build-dependencies]
 cc = { version = "1", default-features = false, features = ["parallel"] }
 either = { version = "1", default-features = false, features = ["use_std"] }
-itertools = { version = "0.11" }
 
 ### END HAKARI SECTION

From 0174078eb93b7fef901820088724343bdace4f54 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 29 Nov 2023 13:41:06 +0800
Subject: [PATCH 177/261] feat: refine rate limiter (#221)

---
 foyer-common/Cargo.toml                       |   4 +-
 foyer-common/src/lib.rs                       |   1 -
 foyer-common/src/rated_random.rs              | 229 ------------------
 foyer-storage-bench/src/main.rs               |  41 +---
 foyer-storage/src/admission/mod.rs            |   1 -
 foyer-storage/src/admission/rated_random.rs   |  68 ------
 foyer-storage/src/admission/rated_ticket.rs   |  39 ++-
 foyer-storage/src/generic.rs                  |  14 --
 foyer-storage/src/lazy.rs                     |   2 -
 foyer-storage/src/reclaimer.rs                |  15 +-
 foyer-storage/src/reinsertion/mod.rs          |   1 -
 foyer-storage/src/reinsertion/rated_random.rs |  68 ------
 foyer-storage/src/reinsertion/rated_ticket.rs |  39 ++-
 foyer-storage/tests/storage_test.rs           |   6 -
 14 files changed, 73 insertions(+), 455 deletions(-)
 delete mode 100644 foyer-common/src/rated_random.rs
 delete mode 100644 foyer-storage/src/admission/rated_random.rs
 delete mode 100644 foyer-storage/src/reinsertion/rated_random.rs

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 0f367928..2f5c17e2 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -17,6 +17,8 @@ foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
-rand = "0.8.5"
 tokio = { workspace = true }
 tracing = "0.1"
+
+[dev-dependencies]
+rand = "0.8.5"
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index e66098dc..491a295e 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -25,6 +25,5 @@ pub mod erwlock;
 pub mod queue;
 pub mod range;
 pub mod rate;
-pub mod rated_random;
 pub mod rated_ticket;
 pub mod runtime;
diff --git a/foyer-common/src/rated_random.rs b/foyer-common/src/rated_random.rs
deleted file mode 100644
index 5ce2460f..00000000
--- a/foyer-common/src/rated_random.rs
+++ /dev/null
@@ -1,229 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{
-    fmt::Debug,
-    sync::atomic::{AtomicUsize, Ordering},
-    time::{Duration, Instant},
-};
-
-use parking_lot::{Mutex, MutexGuard};
-use rand::{thread_rng, Rng};
-
-const PRECISION: usize = 100000;
-
-/// p       : admit probability among △t
-/// w_t    :  entry weight at time t
-/// r       : actual admitted rate
-/// E(w)    : expected admitted weight
-/// E(r)    : expceted admitted rate
-///
-/// E(w_t) = {
-///   p * w  ( if  judge &&  insert )
-///   p * w  ( if !judge && !insert )
-///   w      ( if !judge &&  insert )
-///   0      ( if  judge && !insert )
-/// }
-///
-///    E(r) = sum_{△t}{E(w_t)} / △t
-/// => E(r) * △t = sum_{△t}{ E(w_t) }
-/// => E(r) * △t =   sum_{△t}^{(judge && insert) || (!judge && !insert)}{ p * w_t }
-///                + sum_{△t}^{(!judge && insert)}{ w_t }
-///                + sum_{△t}^{(judge && !insert){ 0 }
-/// => E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t } = p * sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t }
-/// => p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
-///
-/// p = (E(r) * △t) - sum_{△t}^{(!judge && insert)}{ w_t }) / (sum_{△t}^{(judge && insert) || (!judge && !insert)}{ w_t })
-///     ↑ rate        ↑ △force_insert_bytes                   ↑ △obey_bytes
-///
-/// p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
-#[derive(Debug)]
-pub struct RatedRandom {
-    rate: usize,
-    update_interval: Duration,
-
-    obey_bytes: AtomicUsize,
-    force_inser_bytes: AtomicUsize,
-
-    probability: AtomicUsize,
-
-    inner: Mutex<Inner>,
-}
-
-#[derive(Debug)]
-struct Inner {
-    last_update_time: Option<Instant>,
-    last_obey_bytes: usize,
-    last_force_insert_bytes: usize,
-}
-
-impl RatedRandom {
-    pub fn new(rate: usize, update_interval: Duration) -> Self {
-        Self {
-            rate,
-            update_interval,
-
-            obey_bytes: AtomicUsize::new(0),
-            force_inser_bytes: AtomicUsize::new(0),
-
-            probability: AtomicUsize::new(0),
-            inner: Mutex::new(Inner {
-                last_update_time: None,
-                last_obey_bytes: 0,
-                last_force_insert_bytes: 0,
-            }),
-        }
-    }
-
-    pub fn judge(&self) -> bool {
-        if let Some(inner) = self.inner.try_lock() {
-            self.update(inner);
-        }
-
-        thread_rng().gen_range(0..PRECISION) < self.probability.load(Ordering::Relaxed)
-    }
-
-    pub fn on_insert(&self, weight: usize, judge: bool) {
-        if judge {
-            // obey
-            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
-        } else {
-            // force insert
-            self.force_inser_bytes.fetch_add(weight, Ordering::Relaxed);
-        }
-    }
-
-    pub fn on_drop(&self, weight: usize, judge: bool) {
-        if !judge {
-            // obey
-            self.obey_bytes.fetch_add(weight, Ordering::Relaxed);
-        }
-    }
-
-    fn update(&self, mut inner: MutexGuard<'_, Inner>) {
-        // p = ( rate * △t - △force_insert_bytes ) / △obey_bytes
-
-        let now = Instant::now();
-
-        let elapsed = match inner.last_update_time {
-            Some(last_update_time) => now.duration_since(last_update_time),
-            None => self.update_interval,
-        };
-
-        if elapsed < self.update_interval {
-            return;
-        }
-
-        let now_obey_bytes = self.obey_bytes.load(Ordering::Relaxed);
-        let now_force_insert_bytes = self.force_inser_bytes.load(Ordering::Relaxed);
-
-        let delta_obey_bytes = now_obey_bytes - inner.last_obey_bytes;
-        let delta_force_insert_bytes = now_force_insert_bytes - inner.last_force_insert_bytes;
-
-        inner.last_update_time = Some(now);
-        inner.last_obey_bytes = now_obey_bytes;
-        inner.last_force_insert_bytes = now_force_insert_bytes;
-
-        let p = if delta_obey_bytes == 0 {
-            1.0
-        } else {
-            let numerator =
-                self.rate as f64 * elapsed.as_secs_f64() - delta_force_insert_bytes as f64;
-            let numerator = numerator.abs();
-            let p = numerator / delta_obey_bytes as f64;
-            p.min(1.0)
-        };
-
-        debug_assert!((0.0..=1.0).contains(&p), "p out of range 0..=1: {}", p);
-
-        let p = (p * PRECISION as f64) as usize;
-        self.probability.store(p, Ordering::Relaxed);
-
-        tracing::debug!("probability: {}", p as f64 / PRECISION as f64);
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use itertools::Itertools;
-
-    use super::*;
-
-    #[ignore]
-    #[tokio::test]
-    async fn test_rated_random() {
-        const CASES: usize = 10;
-        const ERATIO: f64 = 0.1;
-
-        let handles = (0..CASES).map(|_| tokio::spawn(case())).collect_vec();
-        let mut eratios = vec![];
-        for handle in handles {
-            let eratio = handle.await.unwrap();
-            assert!(eratio < ERATIO, "eratio: {} < ERATIO: {}", eratio, ERATIO);
-            eratios.push(eratio);
-        }
-        println!("========== RatedRandom error ratio begin ==========");
-        for eratio in eratios {
-            println!("eratio: {eratio}");
-        }
-        println!("=========== RatedRandom error ratio end ===========");
-    }
-
-    async fn case() -> f64 {
-        const RATE: usize = 1_000_000;
-        const CONCURRENCY: usize = 10;
-
-        const P_OTHER: f64 = 0.8;
-        const P_FORCE: f64 = 0.1;
-
-        let score = Arc::new(AtomicUsize::new(0));
-
-        let rr = Arc::new(RatedRandom::new(RATE, Duration::from_millis(100)));
-
-        // scope: CONCURRENCY * (1 / interval) * range
-        // [1_000_000, 10_000_000]
-        // FORCE: [100_000, 1_000_000]
-
-        async fn submit(rr: Arc<RatedRandom>, score: Arc<AtomicUsize>) {
-            loop {
-                tokio::time::sleep(Duration::from_millis(1)).await;
-                let weight = thread_rng().gen_range(100..1000);
-
-                let judge = rr.judge();
-                let p_other = thread_rng().gen_range(0.0..=1.0);
-                let p_force = thread_rng().gen_range(0.0..=1.0);
-
-                let insert = (p_force <= P_FORCE) || (p_other <= P_OTHER && judge);
-
-                if insert {
-                    score.fetch_add(weight, Ordering::Relaxed);
-                    rr.on_insert(weight, judge);
-                } else {
-                    rr.on_drop(weight, judge);
-                }
-            }
-        }
-
-        for _ in 0..CONCURRENCY {
-            tokio::spawn(submit(rr.clone(), score.clone()));
-        }
-
-        tokio::time::sleep(Duration::from_secs(10)).await;
-        let s = score.load(Ordering::Relaxed);
-        let error = (s as isize - RATE as isize * 10).unsigned_abs();
-        error as f64 / (RATE as f64 * 10.0)
-    }
-}
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 36415a3b..d48df812 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -38,17 +38,11 @@ use export::MetricsExporter;
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::eviction::lfu::LfuConfig;
 use foyer_storage::{
-    admission::{
-        rated_random::RatedRandomAdmissionPolicy, rated_ticket::RatedTicketAdmissionPolicy,
-        AdmissionPolicy,
-    },
+    admission::{rated_ticket::RatedTicketAdmissionPolicy, AdmissionPolicy},
     compress::Compression,
     device::fs::FsDeviceConfig,
     error::Result,
-    reinsertion::{
-        rated_random::RatedRandomReinsertionPolicy, rated_ticket::RatedTicketReinsertionPolicy,
-        ReinsertionPolicy,
-    },
+    reinsertion::{rated_ticket::RatedTicketReinsertionPolicy, ReinsertionPolicy},
     runtime::{RuntimeConfig, RuntimeStore, RuntimeStoreConfig, RuntimeStoreWriter},
     storage::{Storage, StorageExt, StorageWriter},
     store::{LfuFsStoreConfig, Store, StoreConfig, StoreWriter},
@@ -136,30 +130,16 @@ pub struct Args {
     #[arg(long, default_value_t = 16)]
     recover_concurrency: usize,
 
-    /// enable rated random admission policy if `random_insert_rate_limit` > 0
-    /// (MiB/s)
-    #[arg(long, default_value_t = 0)]
-    random_insert_rate_limit: usize,
-
-    /// enable rated random reinsertion policy if `random_reinsert_rate_limit` > 0
-    /// (MiB/s)
-    #[arg(long, default_value_t = 0)]
-    random_reinsert_rate_limit: usize,
-
     /// enable rated ticket admission policy if `ticket_insert_rate_limit` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     ticket_insert_rate_limit: usize,
 
-    /// enable rated ticket reinsetion policy if `ticket_reinsert_rate_limitgit a` > 0
+    /// enable rated ticket reinsetion policy if `ticket_reinsert_rate_limit` > 0
     /// (MiB/s)
     #[arg(long, default_value_t = 0)]
     ticket_reinsert_rate_limit: usize,
 
-    /// (MiB/s)
-    #[arg(long, default_value_t = 0)]
-    reclaim_rate_limit: usize,
-
     /// `0` means equal to reclaimer count
     #[arg(long, default_value_t = 0)]
     clean_region_threshold: usize,
@@ -525,20 +505,6 @@ async fn main() {
 
     let mut admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
     let mut reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
-    if args.random_insert_rate_limit > 0 {
-        let rr = RatedRandomAdmissionPolicy::new(
-            args.random_insert_rate_limit * 1024 * 1024,
-            Duration::from_millis(100),
-        );
-        admissions.push(Arc::new(rr));
-    }
-    if args.random_reinsert_rate_limit > 0 {
-        let rr = RatedRandomReinsertionPolicy::new(
-            args.random_reinsert_rate_limit * 1024 * 1024,
-            Duration::from_millis(100),
-        );
-        reinsertions.push(Arc::new(rr));
-    }
     if args.ticket_insert_rate_limit > 0 {
         let rt = RatedTicketAdmissionPolicy::new(args.ticket_insert_rate_limit * 1024 * 1024);
         admissions.push(Arc::new(rt));
@@ -571,7 +537,6 @@ async fn main() {
         flusher_buffer_size: args.flusher_buffer_size,
         flushers: args.flushers,
         reclaimers: args.reclaimers,
-        reclaim_rate_limit: args.reclaim_rate_limit * 1024 * 1024,
         recover_concurrency: args.recover_concurrency,
         clean_region_threshold,
         compression,
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 3912929f..2cdeb5ef 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -41,5 +41,4 @@ pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     fn on_drop(&self, key: &Self::Key, weight: usize, judge: bool);
 }
 
-pub mod rated_random;
 pub mod rated_ticket;
diff --git a/foyer-storage/src/admission/rated_random.rs b/foyer-storage/src/admission/rated_random.rs
deleted file mode 100644
index 32702df7..00000000
--- a/foyer-storage/src/admission/rated_random.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{fmt::Debug, marker::PhantomData, time::Duration};
-
-use foyer_common::{
-    code::{Key, Value},
-    rated_random::RatedRandom,
-};
-
-use super::AdmissionPolicy;
-
-#[derive(Debug)]
-pub struct RatedRandomAdmissionPolicy<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    inner: RatedRandom,
-
-    _marker: PhantomData<(K, V)>,
-}
-
-impl<K, V> RatedRandomAdmissionPolicy<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    pub fn new(rate: usize, update_interval: Duration) -> Self {
-        Self {
-            inner: RatedRandom::new(rate, update_interval),
-            _marker: PhantomData,
-        }
-    }
-}
-
-impl<K, V> AdmissionPolicy for RatedRandomAdmissionPolicy<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Key = K;
-
-    type Value = V;
-
-    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
-        self.inner.judge()
-    }
-
-    fn on_insert(&self, _key: &Self::Key, weight: usize, judge: bool) {
-        self.inner.on_insert(weight, judge)
-    }
-
-    fn on_drop(&self, _key: &Self::Key, weight: usize, judge: bool) {
-        self.inner.on_drop(weight, judge)
-    }
-}
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
index c81f62a8..2dbab8d3 100644
--- a/foyer-storage/src/admission/rated_ticket.rs
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -12,14 +12,21 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData};
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        OnceLock,
+    },
+};
 
 use foyer_common::{
     code::{Key, Value},
     rated_ticket::RatedTicket,
 };
 
-use super::AdmissionPolicy;
+use super::{AdmissionContext, AdmissionPolicy};
 
 #[derive(Debug)]
 pub struct RatedTicketAdmissionPolicy<K, V>
@@ -29,6 +36,10 @@ where
 {
     inner: RatedTicket,
 
+    last: AtomicUsize,
+
+    context: OnceLock<AdmissionContext<K>>,
+
     _marker: PhantomData<(K, V)>,
 }
 
@@ -40,6 +51,8 @@ where
     pub fn new(rate: usize) -> Self {
         Self {
             inner: RatedTicket::new(rate as f64),
+            last: AtomicUsize::default(),
+            context: OnceLock::new(),
             _marker: PhantomData,
         }
     }
@@ -54,13 +67,27 @@ where
 
     type Value = V;
 
-    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
-        self.inner.probe()
+    fn init(&self, context: AdmissionContext<Self::Key>) {
+        self.context.set(context).unwrap();
     }
 
-    fn on_insert(&self, _key: &Self::Key, weight: usize, _judge: bool) {
-        self.inner.reduce(weight as f64);
+    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
+        let res = self.inner.probe();
+
+        let metrics = self.context.get().unwrap().metrics.as_ref();
+        let current = metrics.op_bytes_flush.get() as usize;
+        let last = self.last.load(Ordering::Relaxed);
+        let delta = current.saturating_sub(last);
+
+        if delta > 0 {
+            self.last.store(current, Ordering::Relaxed);
+            self.inner.reduce(delta as f64);
+        }
+
+        res
     }
 
+    fn on_insert(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
+
     fn on_drop(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 }
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index e748807a..b2743bf8 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -29,7 +29,6 @@ use bytes::{Buf, BufMut};
 use foyer_common::{
     bits,
     code::{CodingError, Key, Value},
-    rate::RateLimiter,
 };
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use futures::future::try_join_all;
@@ -99,9 +98,6 @@ where
     /// Count of reclaimers.
     pub reclaimers: usize,
 
-    /// Flush rate limits.
-    pub reclaim_rate_limit: usize,
-
     /// Clean region count threshold to trigger reclamation.
     ///
     /// `clean_region_threshold` is recommended to be equal or larger than `reclaimers`.
@@ -132,7 +128,6 @@ where
             .field("flusher_buffer_size", &self.flusher_buffer_size)
             .field("flushers", &self.flushers)
             .field("reclaimers", &self.reclaimers)
-            .field("reclaim_rate_limit", &self.reclaim_rate_limit)
             .field("clean_region_threshold", &self.clean_region_threshold)
             .field("recover_concurrency", &self.recover_concurrency)
             .field("compression", &self.compression)
@@ -159,7 +154,6 @@ where
             flusher_buffer_size: self.flusher_buffer_size,
             flushers: self.flushers,
             reclaimers: self.reclaimers,
-            reclaim_rate_limit: self.reclaim_rate_limit,
             clean_region_threshold: self.clean_region_threshold,
             recover_concurrency: self.recover_concurrency,
             compression: self.compression,
@@ -316,11 +310,6 @@ where
             reinsertion.init(reinsertion_context.clone());
         }
 
-        let reclaim_rate_limiter = match config.reclaim_rate_limit {
-            0 => None,
-            rate => Some(Arc::new(RateLimiter::new(rate as f64))),
-        };
-
         let flushers = flusher_stop_rxs
             .into_iter()
             .zip_eq(flusher_entry_rxs.into_iter())
@@ -344,7 +333,6 @@ where
                     config.clean_region_threshold,
                     store.clone(),
                     region_manager.clone(),
-                    reclaim_rate_limiter.clone(),
                     metrics.clone(),
                     stop_rx,
                 )
@@ -1194,7 +1182,6 @@ mod tests {
             flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
-            reclaim_rate_limit: 0,
             recover_concurrency: 2,
             clean_region_threshold: 1,
             compression: Compression::None,
@@ -1246,7 +1233,6 @@ mod tests {
             flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 0,
-            reclaim_rate_limit: 0,
             recover_concurrency: 2,
             clean_region_threshold: 1,
             compression: Compression::None,
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 8354c9ee..c64b9db4 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -260,7 +260,6 @@ mod tests {
             flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
-            reclaim_rate_limit: 0,
             recover_concurrency: 2,
             clean_region_threshold: 1,
             compression: crate::compress::Compression::None,
@@ -295,7 +294,6 @@ mod tests {
             flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
-            reclaim_rate_limit: 0,
             recover_concurrency: 2,
             clean_region_threshold: 1,
             compression: crate::compress::Compression::None,
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 77b5f2b9..d35f4830 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -18,10 +18,7 @@ use std::{
 };
 
 use bytes::BufMut;
-use foyer_common::{
-    code::{Key, Value},
-    rate::RateLimiter,
-};
+use foyer_common::code::{Key, Value};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use tokio::sync::broadcast;
 
@@ -50,8 +47,6 @@ where
 
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
-    rate_limiter: Option<Arc<RateLimiter>>,
-
     metrics: Arc<Metrics>,
 
     stop_rx: broadcast::Receiver<()>,
@@ -69,7 +64,6 @@ where
         threshold: usize,
         store: GenericStore<K, V, D, EP, EL>,
         region_manager: Arc<RegionManager<D, EP, EL>>,
-        rate_limiter: Option<Arc<RateLimiter>>,
         metrics: Arc<Metrics>,
         stop_rx: broadcast::Receiver<()>,
     ) -> Self {
@@ -77,7 +71,6 @@ where
             threshold,
             store,
             region_manager,
-            rate_limiter,
             metrics,
             stop_rx,
         }
@@ -135,7 +128,6 @@ where
         let reinsert = || {
             let region = region.clone();
             let metrics = self.metrics.clone();
-            let rate = self.rate_limiter.clone();
             let reinsertions = self.store.reinsertions().clone();
 
             tracing::info!("[reclaimer] begin reinsertion, region: {}", region_id);
@@ -163,11 +155,6 @@ where
                         continue;
                     }
 
-                    // TODO(MrCroxx): Should reclaimer use wait if exceed limitation?
-                    if let Some(rate) = rate.as_ref() && let Some(wait) = rate.consume(weight as f64) {
-                        tokio::time::sleep(wait).await;
-                    }
-
                     let mut writer = self.store.writer(key.clone(), weight);
                     writer.set_skippable();
 
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index b4fc9be2..6d4c32c4 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -42,5 +42,4 @@ pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
 }
 
 pub mod exist;
-pub mod rated_random;
 pub mod rated_ticket;
diff --git a/foyer-storage/src/reinsertion/rated_random.rs b/foyer-storage/src/reinsertion/rated_random.rs
deleted file mode 100644
index 5eb99386..00000000
--- a/foyer-storage/src/reinsertion/rated_random.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{fmt::Debug, marker::PhantomData, time::Duration};
-
-use foyer_common::{
-    code::{Key, Value},
-    rated_random::RatedRandom,
-};
-
-use super::ReinsertionPolicy;
-
-#[derive(Debug)]
-pub struct RatedRandomReinsertionPolicy<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    inner: RatedRandom,
-
-    _marker: PhantomData<(K, V)>,
-}
-
-impl<K, V> RatedRandomReinsertionPolicy<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    pub fn new(rate: usize, update_interval: Duration) -> Self {
-        Self {
-            inner: RatedRandom::new(rate, update_interval),
-            _marker: PhantomData,
-        }
-    }
-}
-
-impl<K, V> ReinsertionPolicy for RatedRandomReinsertionPolicy<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Key = K;
-
-    type Value = V;
-
-    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
-        self.inner.judge()
-    }
-
-    fn on_insert(&self, _key: &Self::Key, weight: usize, judge: bool) {
-        self.inner.on_insert(weight, judge)
-    }
-
-    fn on_drop(&self, _key: &Self::Key, weight: usize, judge: bool) {
-        self.inner.on_drop(weight, judge)
-    }
-}
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
index 74d70ba1..afebaf02 100644
--- a/foyer-storage/src/reinsertion/rated_ticket.rs
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -12,14 +12,21 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData};
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        OnceLock,
+    },
+};
 
 use foyer_common::{
     code::{Key, Value},
     rated_ticket::RatedTicket,
 };
 
-use super::ReinsertionPolicy;
+use super::{ReinsertionContext, ReinsertionPolicy};
 
 #[derive(Debug)]
 pub struct RatedTicketReinsertionPolicy<K, V>
@@ -29,6 +36,10 @@ where
 {
     inner: RatedTicket,
 
+    last: AtomicUsize,
+
+    context: OnceLock<ReinsertionContext<K>>,
+
     _marker: PhantomData<(K, V)>,
 }
 
@@ -40,6 +51,8 @@ where
     pub fn new(rate: usize) -> Self {
         Self {
             inner: RatedTicket::new(rate as f64),
+            last: AtomicUsize::default(),
+            context: OnceLock::new(),
             _marker: PhantomData,
         }
     }
@@ -54,13 +67,27 @@ where
 
     type Value = V;
 
-    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
-        self.inner.probe()
+    fn init(&self, context: super::ReinsertionContext<Self::Key>) {
+        self.context.set(context).unwrap();
     }
 
-    fn on_insert(&self, _key: &Self::Key, weight: usize, _judge: bool) {
-        self.inner.reduce(weight as f64);
+    fn judge(&self, _key: &Self::Key, _weight: usize) -> bool {
+        let res = self.inner.probe();
+
+        let metrics = self.context.get().unwrap().metrics.as_ref();
+        let current = metrics.op_bytes_reinsert.get() as usize;
+        let last = self.last.load(Ordering::Relaxed);
+        let delta = current.saturating_sub(last);
+
+        if delta > 0 {
+            self.last.store(current, Ordering::Relaxed);
+            self.inner.reduce(delta as f64);
+        }
+
+        res
     }
 
+    fn on_insert(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
+
     fn on_drop(&self, _key: &Self::Key, _weight: usize, _judge: bool) {}
 }
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 07fac38f..72f9aa8b 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -137,7 +137,6 @@ async fn test_store() {
         flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
-        reclaim_rate_limit: 0,
         clean_region_threshold: 1,
         recover_concurrency: 2,
         compression: Compression::None,
@@ -167,7 +166,6 @@ async fn test_store_zstd() {
         flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
-        reclaim_rate_limit: 0,
         clean_region_threshold: 1,
         recover_concurrency: 2,
         compression: Compression::Zstd,
@@ -197,7 +195,6 @@ async fn test_store_lz4() {
         flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
-        reclaim_rate_limit: 0,
         clean_region_threshold: 1,
         recover_concurrency: 2,
         compression: Compression::Lz4,
@@ -227,7 +224,6 @@ async fn test_lazy_store() {
         flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
-        reclaim_rate_limit: 0,
         clean_region_threshold: 1,
         recover_concurrency: 2,
         compression: Compression::None,
@@ -258,7 +254,6 @@ async fn test_runtime_store() {
             flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
-            reclaim_rate_limit: 0,
             clean_region_threshold: 1,
             recover_concurrency: 2,
             compression: Compression::None,
@@ -295,7 +290,6 @@ async fn test_runtime_lazy_store() {
             flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
-            reclaim_rate_limit: 0,
             clean_region_threshold: 1,
             recover_concurrency: 2,
             compression: Compression::None,

From 75d79bc25a59b0f6a36a8e3eabbd30c3690f410e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 29 Nov 2023 14:28:16 +0800
Subject: [PATCH 178/261] chore: bump foyer to 0.2.0 (#223)

---
 foyer-common/Cargo.toml         |  4 +++-
 foyer-intrusive/Cargo.toml      |  6 ++++--
 foyer-memory/Cargo.toml         |  4 +++-
 foyer-storage-bench/Cargo.toml  | 10 ++++++----
 foyer-storage/Cargo.toml        |  8 +++++---
 foyer-workspace-hack/Cargo.toml |  6 +++++-
 foyer/Cargo.toml                | 10 ++++++----
 7 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 2f5c17e2..055acb85 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -3,8 +3,10 @@ name = "foyer-common"
 version = "0.1.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
+description = "common utils for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 8bfe7b11..26980ace 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -3,8 +3,10 @@ name = "foyer-intrusive"
 version = "0.1.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
+description = "intrusive data structures for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
@@ -13,7 +15,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { path = "../foyer-common" }
+foyer-common = { version = "0.1", path = "../foyer-common" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 memoffset = "0.9"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 2dd2de38..1ca7f47c 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -3,8 +3,10 @@ name = "foyer-memory"
 version = "0.1.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
+description = "memory cache for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 6c2707a7..119b5b11 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -3,8 +3,10 @@ name = "foyer-storage-bench"
 version = "0.1.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
+description = "storage engine bench tool for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
@@ -15,9 +17,9 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
-foyer-common = { path = "../foyer-common" }
-foyer-intrusive = { path = "../foyer-intrusive" }
-foyer-storage = { path = "../foyer-storage" }
+foyer-common = { version = "0.1", path = "../foyer-common" }
+foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.1", path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index e1332f8d..dc6c68e1 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -3,8 +3,10 @@ name = "foyer-storage"
 version = "0.1.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
+description = "storage engine for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
@@ -16,8 +18,8 @@ bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { path = "../foyer-common" }
-foyer-intrusive = { path = "../foyer-intrusive" }
+foyer-common = { version = "0.1", path = "../foyer-common" }
+foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.12"
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 005f546b..6d6f2491 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -5,9 +5,13 @@
 [package]
 name = "foyer-workspace-hack"
 version = "0.1.0"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "workspace-hack package, managed by hakari"
+license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
 # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.
-publish = false
+publish = true
 # The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments
 # are managed by hakari.
 
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 675477f7..51335401 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,17 +1,19 @@
 [package]
 name = "foyer"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
 license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
-foyer-common = { path = "../foyer-common" }
-foyer-intrusive = { path = "../foyer-intrusive" }
-foyer-storage = { path = "../foyer-storage" }
+foyer-common = { version = "0.1", path = "../foyer-common" }
+foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.1", path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }

From fb5cd5eaa996e0cb48e6a5e8d954a4b902f45ede Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 4 Dec 2023 15:30:41 +0800
Subject: [PATCH 179/261] feat: add format version in region header (#224)

---
 foyer-storage/src/buffer.rs  |  3 +-
 foyer-storage/src/generic.rs |  9 ++----
 foyer-storage/src/region.rs  | 57 ++++++++++++++++++++++++++++++++----
 3 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 6028d29b..5fcc96e0 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -22,7 +22,7 @@ use crate::{
     device::{error::DeviceError, Device},
     flusher::Entry,
     generic::{checksum, EntryHeader},
-    region::{RegionHeader, RegionId, REGION_MAGIC},
+    region::{RegionHeader, RegionId, Version, REGION_MAGIC},
 };
 
 #[derive(thiserror::Error, Debug)]
@@ -113,6 +113,7 @@ where
         unsafe { self.buffer.set_len(self.device.align()) };
         let header = RegionHeader {
             magic: REGION_MAGIC,
+            version: Version::latest(),
         };
         header.write(&mut self.buffer[..]);
         debug_assert_eq!(self.buffer.len(), self.device.align());
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index b2743bf8..625efa29 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -50,7 +50,7 @@ use crate::{
     judge::Judges,
     metrics::{Metrics, METRICS},
     reclaimer::Reclaimer,
-    region::{Region, RegionHeader, RegionId, REGION_MAGIC},
+    region::{Region, RegionHeader, RegionId},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::{ReinsertionContext, ReinsertionPolicy},
     ring::RingBuffer,
@@ -933,12 +933,9 @@ where
             None => return Ok(None),
         };
 
-        let header = RegionHeader::read(slice.as_ref());
-        drop(slice);
-
-        if header.magic != REGION_MAGIC {
+        let Ok(_) = RegionHeader::read(slice.as_ref()) else {
             return Ok(None);
-        }
+        };
 
         Ok(Some(Self {
             region,
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 38491ab0..5f519df0 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -36,20 +36,67 @@ pub type RegionId = u32;
 
 pub const REGION_MAGIC: u64 = 0x19970327;
 
+#[derive(Debug)]
+pub enum Version {
+    V1,
+}
+
+impl Version {
+    pub fn latest() -> Self {
+        Self::V1
+    }
+
+    pub fn to_u64(&self) -> u64 {
+        match self {
+            Version::V1 => 1,
+        }
+    }
+}
+
+impl From<Version> for u64 {
+    fn from(value: Version) -> Self {
+        match value {
+            Version::V1 => 1,
+        }
+    }
+}
+
+impl TryFrom<u64> for Version {
+    type Error = anyhow::Error;
+
+    fn try_from(value: u64) -> std::result::Result<Self, Self::Error> {
+        match value {
+            1 => Ok(Self::V1),
+            v => Err(anyhow::anyhow!("invalid region format version: {}", v)),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct RegionHeader {
     /// magic number to decide a valid region
     pub magic: u64,
+    /// format version
+    pub version: Version,
 }
 
 impl RegionHeader {
-    pub fn write(&self, buf: &mut [u8]) {
-        (&mut buf[..]).put_u64(self.magic);
+    pub fn write(&self, mut buf: &mut [u8]) {
+        buf.put_u64(self.magic);
+        buf.put_u64(self.version.to_u64());
     }
 
-    pub fn read(buf: &[u8]) -> Self {
-        let magic = (&buf[..]).get_u64();
-        Self { magic }
+    pub fn read(mut buf: &[u8]) -> std::result::Result<Self, anyhow::Error> {
+        let magic = buf.get_u64();
+        if magic != REGION_MAGIC {
+            return Err(anyhow::anyhow!(
+                "region magic mismatch, magic: {}, expected: {}",
+                magic,
+                REGION_MAGIC
+            ));
+        }
+        let version = buf.get_u64().try_into()?;
+        Ok(Self { magic, version })
     }
 }
 

From 4d38f4aa5512de5d54ba7ce808958202abbcf677 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 12 Dec 2023 17:02:40 +0800
Subject: [PATCH 180/261] refactor: introduce cursor for serialization (#227)

---
 foyer-common/src/code.rs | 228 ++++++++++++++++++++++++++++++++++++++-
 foyer-common/src/lib.rs  |   2 +
 2 files changed, 226 insertions(+), 4 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 3755ea3f..81806205 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -12,12 +12,94 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::marker::PhantomData;
+
 use bytes::{Buf, BufMut};
 use paste::paste;
 
 pub type CodingError = anyhow::Error;
 pub type CodingResult<T> = Result<T, CodingError>;
 
+trait BufMutExt: BufMut {
+    cfg_match! {
+        cfg(target_pointer_width = "16") => {
+            fn put_usize(&mut self, v: usize) {
+                self.put_u16(v as u16);
+            }
+
+            fn put_isize(&mut self, v: isize) {
+                self.put_i16(v as i16);
+            }
+        }
+        cfg(target_pointer_width = "32") => {
+            fn put_usize(&mut self, v: usize) {
+                self.put_u32(v as u32);
+            }
+
+            fn put_isize(&mut self, v: isize) {
+                self.put_i32(v as i32);
+            }
+        }
+        cfg(target_pointer_width = "64") => {
+            fn put_usize(&mut self, v: usize) {
+                self.put_u64(v as u64);
+            }
+
+            fn put_isize(&mut self, v: isize) {
+                self.put_i64(v as i64);
+            }
+        }
+    }
+}
+
+impl<T: Buf> BufExt for T {}
+
+trait BufExt: Buf {
+    cfg_match! {
+        cfg(target_pointer_width = "16") => {
+            fn get_usize(&mut self) -> usize {
+                self.get_u16() as usize
+            }
+
+            fn get_isize(&mut self) -> isize {
+                self.get_i16() as isize
+            }
+        }
+        cfg(target_pointer_width = "32") => {
+            fn get_usize(&mut self) -> usize {
+                self.get_u32() as usize
+            }
+
+            fn get_isize(&mut self) -> isize {
+                self.get_i32() as isize
+            }
+        }
+        cfg(target_pointer_width = "64") => {
+            fn get_usize(&mut self) -> usize {
+                self.get_u64() as usize
+            }
+
+            fn get_isize(&mut self) -> isize {
+                self.get_i64() as isize
+            }
+        }
+    }
+}
+
+impl<T: BufMut> BufMutExt for T {}
+
+pub trait Cursor: Send + Sync + 'static + std::io::Read {
+    type T: Send + Sync + 'static;
+
+    fn inner(&self) -> &Self::T;
+
+    fn len(&self) -> usize;
+
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
 #[expect(unused_variables)]
 pub trait Key:
     Sized
@@ -32,6 +114,9 @@ pub trait Key:
     + Clone
     + std::fmt::Debug
 {
+    type Cursor: Cursor<T = Self> = UnimplementedCursor<Self>;
+
+    /// memory weight
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
@@ -47,10 +132,17 @@ pub trait Key:
     fn read(buf: &[u8]) -> CodingResult<Self> {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
+
+    fn into_cursor(self) -> Self::Cursor {
+        panic!("Associated type `Cursor` and method `into_cursor` must be implemented for `Key` if storage is used.")
+    }
 }
 
 #[expect(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
+    type Cursor: Cursor<T = Self> = UnimplementedCursor<Self>;
+
+    /// memory weight
     fn weight(&self) -> usize {
         std::mem::size_of::<Self>()
     }
@@ -66,22 +158,70 @@ pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
     fn read(buf: &[u8]) -> CodingResult<Self> {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
+
+    fn into_cursor(self) -> Self::Cursor {
+        panic!("Associated type `Cursor` and method `into_cursor` must be implemented for `Value` if storage is used.")
+    }
 }
 
 macro_rules! for_all_primitives {
     ($macro:ident) => {
         $macro! {
-            u8, u16, u32, u64,
-            i8, i16, i32, i64,
+            {u8, U8},
+            {u16, U16},
+            {u32, U32},
+            {u64, U64},
+            {usize, Usize},
+            {i8, I8},
+            {i16, I16},
+            {i32, I32},
+            {i64, I64},
+            {isize, Isize},
+        }
+    };
+}
+
+macro_rules! def_cursor {
+    ($( { $type:ty, $id:ident }, )*) => {
+        paste! {
+            $(
+                pub struct [<PrimitiveCursor $id>] {
+                    inner: $type,
+                    pos: u8,
+                }
+
+                impl std::io::Read for [<PrimitiveCursor $id>] {
+                    fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
+                        let slice = self.inner.to_be_bytes();
+                        let len = std::cmp::min(slice.len() - self.pos as usize, buf.len());
+                        buf.put_slice(&slice[self.pos as usize..self.pos as usize + len]);
+                        self.pos += len as u8;
+                        Ok(len)
+                    }
+                }
+
+                impl Cursor for [<PrimitiveCursor $id>] {
+                    type T = $type;
+
+                    fn inner(&self) -> &Self::T {
+                        &self.inner
+                    }
+
+                    fn len(&self) -> usize {
+                        std::mem::size_of::<$type>()
+                    }
+                }
+            )*
         }
     };
 }
 
 macro_rules! impl_key {
-    ($( $type:ty, )*) => {
+    ($( { $type:ty, $id:ident }, )*) => {
         paste! {
             $(
                 impl Key for $type {
+                    type Cursor = [<PrimitiveCursor $id>];
 
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
@@ -97,6 +237,13 @@ macro_rules! impl_key {
                     fn read(mut buf: &[u8]) -> CodingResult<Self> {
                         Ok(buf.[< get_ $type>]())
                     }
+
+                    fn into_cursor(self) -> Self::Cursor {
+                        [<PrimitiveCursor $id>] {
+                            inner: self,
+                            pos: 0,
+                        }
+                    }
                 }
             )*
         }
@@ -104,10 +251,11 @@ macro_rules! impl_key {
 }
 
 macro_rules! impl_value {
-    ($( $type:ty, )*) => {
+    ($( { $type:ty, $id:ident }, )*) => {
         paste! {
             $(
                 impl Value for $type {
+                    type Cursor = [<PrimitiveCursor $id>];
 
                     fn serialized_len(&self) -> usize {
                         std::mem::size_of::<$type>()
@@ -123,12 +271,20 @@ macro_rules! impl_value {
                     fn read(mut buf: &[u8]) -> CodingResult<Self> {
                         Ok(buf.[< get_ $type>]())
                     }
+
+                    fn into_cursor(self) -> Self::Cursor {
+                        [<PrimitiveCursor $id>] {
+                            inner: self,
+                            pos: 0,
+                        }
+                    }
                 }
             )*
         }
     };
 }
 
+for_all_primitives! { def_cursor }
 for_all_primitives! { impl_key }
 for_all_primitives! { impl_value }
 
@@ -152,6 +308,8 @@ impl Key for Vec<u8> {
 }
 
 impl Value for Vec<u8> {
+    type Cursor = std::io::Cursor<Vec<u8>>;
+
     fn weight(&self) -> usize {
         self.len()
     }
@@ -168,6 +326,42 @@ impl Value for Vec<u8> {
     fn read(buf: &[u8]) -> CodingResult<Self> {
         Ok(buf.to_vec())
     }
+
+    fn into_cursor(self) -> Self::Cursor {
+        std::io::Cursor::new(self)
+    }
+}
+
+impl Cursor for std::io::Cursor<Vec<u8>> {
+    type T = Vec<u8>;
+
+    fn inner(&self) -> &Self::T {
+        self.get_ref()
+    }
+
+    fn len(&self) -> usize {
+        self.get_ref().len()
+    }
+}
+
+pub struct PrimitiveCursorVoid;
+
+impl std::io::Read for PrimitiveCursorVoid {
+    fn read(&mut self, _buf: &mut [u8]) -> std::io::Result<usize> {
+        Ok(0)
+    }
+}
+
+impl Cursor for PrimitiveCursorVoid {
+    type T = ();
+
+    fn inner(&self) -> &Self::T {
+        &()
+    }
+
+    fn len(&self) -> usize {
+        0
+    }
 }
 
 impl Key for () {
@@ -189,6 +383,8 @@ impl Key for () {
 }
 
 impl Value for () {
+    type Cursor = PrimitiveCursorVoid;
+
     fn weight(&self) -> usize {
         0
     }
@@ -204,4 +400,28 @@ impl Value for () {
     fn read(_buf: &[u8]) -> CodingResult<Self> {
         Ok(())
     }
+
+    fn into_cursor(self) -> Self::Cursor {
+        PrimitiveCursorVoid
+    }
+}
+
+pub struct UnimplementedCursor<T: Send + Sync + 'static>(PhantomData<T>);
+
+impl<T: Send + Sync + 'static> std::io::Read for UnimplementedCursor<T> {
+    fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
+        unimplemented!()
+    }
+}
+
+impl<T: Send + Sync + 'static> Cursor for UnimplementedCursor<T> {
+    type T = T;
+
+    fn inner(&self) -> &Self::T {
+        unimplemented!()
+    }
+
+    fn len(&self) -> usize {
+        unimplemented!()
+    }
 }
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 491a295e..1d5c401f 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -15,7 +15,9 @@
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
 #![feature(bound_map)]
+#![feature(associated_type_defaults)]
 #![cfg_attr(coverage_nightly, feature(coverage_attribute))]
+#![feature(cfg_match)]
 
 pub mod batch;
 pub mod bits;

From d85af3e0c05ce8cc92777954a19cde6cca507028 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 12 Dec 2023 18:38:23 +0800
Subject: [PATCH 181/261] refactor: require Key and Value to impl Clone (#228)

---
 foyer-common/src/code.rs             | 78 ++++++++++++++++++----------
 foyer-memory/src/lib.rs              |  2 +-
 foyer-storage/src/admission/mod.rs   | 14 ++++-
 foyer-storage/src/reinsertion/mod.rs | 14 ++++-
 4 files changed, 77 insertions(+), 31 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 81806205..194b548e 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -20,33 +20,33 @@ use paste::paste;
 pub type CodingError = anyhow::Error;
 pub type CodingResult<T> = Result<T, CodingError>;
 
-trait BufMutExt: BufMut {
+trait BufExt: Buf {
     cfg_match! {
         cfg(target_pointer_width = "16") => {
-            fn put_usize(&mut self, v: usize) {
-                self.put_u16(v as u16);
+            fn get_usize(&mut self) -> usize {
+                self.get_u16() as usize
             }
 
-            fn put_isize(&mut self, v: isize) {
-                self.put_i16(v as i16);
+            fn get_isize(&mut self) -> isize {
+                self.get_i16() as isize
             }
         }
         cfg(target_pointer_width = "32") => {
-            fn put_usize(&mut self, v: usize) {
-                self.put_u32(v as u32);
+            fn get_usize(&mut self) -> usize {
+                self.get_u32() as usize
             }
 
-            fn put_isize(&mut self, v: isize) {
-                self.put_i32(v as i32);
+            fn get_isize(&mut self) -> isize {
+                self.get_i32() as isize
             }
         }
         cfg(target_pointer_width = "64") => {
-            fn put_usize(&mut self, v: usize) {
-                self.put_u64(v as u64);
+            fn get_usize(&mut self) -> usize {
+                self.get_u64() as usize
             }
 
-            fn put_isize(&mut self, v: isize) {
-                self.put_i64(v as i64);
+            fn get_isize(&mut self) -> isize {
+                self.get_i64() as isize
             }
         }
     }
@@ -54,33 +54,33 @@ trait BufMutExt: BufMut {
 
 impl<T: Buf> BufExt for T {}
 
-trait BufExt: Buf {
+trait BufMutExt: BufMut {
     cfg_match! {
         cfg(target_pointer_width = "16") => {
-            fn get_usize(&mut self) -> usize {
-                self.get_u16() as usize
+            fn put_usize(&mut self, v: usize) {
+                self.put_u16(v as u16);
             }
 
-            fn get_isize(&mut self) -> isize {
-                self.get_i16() as isize
+            fn put_isize(&mut self, v: isize) {
+                self.put_i16(v as i16);
             }
         }
         cfg(target_pointer_width = "32") => {
-            fn get_usize(&mut self) -> usize {
-                self.get_u32() as usize
+            fn put_usize(&mut self, v: usize) {
+                self.put_u32(v as u32);
             }
 
-            fn get_isize(&mut self) -> isize {
-                self.get_i32() as isize
+            fn put_isize(&mut self, v: isize) {
+                self.put_i32(v as i32);
             }
         }
         cfg(target_pointer_width = "64") => {
-            fn get_usize(&mut self) -> usize {
-                self.get_u64() as usize
+            fn put_usize(&mut self, v: usize) {
+                self.put_u64(v as u64);
             }
 
-            fn get_isize(&mut self) -> isize {
-                self.get_i64() as isize
+            fn put_isize(&mut self, v: isize) {
+                self.put_i64(v as i64);
             }
         }
     }
@@ -93,6 +93,8 @@ pub trait Cursor: Send + Sync + 'static + std::io::Read {
 
     fn inner(&self) -> &Self::T;
 
+    fn into_inner(self) -> Self::T;
+
     fn len(&self) -> usize;
 
     fn is_empty(&self) -> bool {
@@ -100,6 +102,9 @@ pub trait Cursor: Send + Sync + 'static + std::io::Read {
     }
 }
 
+/// [`Key`] is required to implement [`Clone`].
+///
+/// If cloning a [`Key`] is expensive, wrap it with [`std::sync::Arc`].
 #[expect(unused_variables)]
 pub trait Key:
     Sized
@@ -111,8 +116,8 @@ pub trait Key:
     + PartialEq
     + Ord
     + PartialOrd
-    + Clone
     + std::fmt::Debug
+    + Clone
 {
     type Cursor: Cursor<T = Self> = UnimplementedCursor<Self>;
 
@@ -138,8 +143,11 @@ pub trait Key:
     }
 }
 
+/// [`Value`] is required to implement [`Clone`].
+///
+/// If cloning a [`Value`] is expensive, wrap it with [`std::sync::Arc`].
 #[expect(unused_variables)]
-pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug {
+pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug + Clone {
     type Cursor: Cursor<T = Self> = UnimplementedCursor<Self>;
 
     /// memory weight
@@ -207,6 +215,10 @@ macro_rules! def_cursor {
                         &self.inner
                     }
 
+                    fn into_inner(self) -> Self::T {
+                        self.inner
+                    }
+
                     fn len(&self) -> usize {
                         std::mem::size_of::<$type>()
                     }
@@ -339,6 +351,10 @@ impl Cursor for std::io::Cursor<Vec<u8>> {
         self.get_ref()
     }
 
+    fn into_inner(self) -> Self::T {
+        self.into_inner()
+    }
+
     fn len(&self) -> usize {
         self.get_ref().len()
     }
@@ -359,6 +375,8 @@ impl Cursor for PrimitiveCursorVoid {
         &()
     }
 
+    fn into_inner(self) -> Self::T {}
+
     fn len(&self) -> usize {
         0
     }
@@ -421,6 +439,10 @@ impl<T: Send + Sync + 'static> Cursor for UnimplementedCursor<T> {
         unimplemented!()
     }
 
+    fn into_inner(self) -> Self::T {
+        unimplemented!()
+    }
+
     fn len(&self) -> usize {
         unimplemented!()
     }
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index b78bb7b1..93aabd63 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -218,7 +218,7 @@ mod tests {
         }
     }
 
-    #[derive(Debug, PartialEq, Eq)]
+    #[derive(Debug, PartialEq, Eq, Clone)]
     struct V(usize);
 
     impl Value for V {
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 2cdeb5ef..8a90724a 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -18,7 +18,7 @@ use foyer_common::code::{Key, Value};
 
 use crate::{catalog::Catalog, metrics::Metrics};
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct AdmissionContext<K>
 where
     K: Key,
@@ -27,6 +27,18 @@ where
     pub metrics: Arc<Metrics>,
 }
 
+impl<K> Clone for AdmissionContext<K>
+where
+    K: Key,
+{
+    fn clone(&self) -> Self {
+        Self {
+            catalog: self.catalog.clone(),
+            metrics: self.metrics.clone(),
+        }
+    }
+}
+
 #[expect(unused_variables)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index 6d4c32c4..afb294e2 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -18,7 +18,7 @@ use foyer_common::code::{Key, Value};
 
 use crate::{catalog::Catalog, metrics::Metrics};
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct ReinsertionContext<K>
 where
     K: Key,
@@ -27,6 +27,18 @@ where
     pub metrics: Arc<Metrics>,
 }
 
+impl<K> Clone for ReinsertionContext<K>
+where
+    K: Key,
+{
+    fn clone(&self) -> Self {
+        Self {
+            catalog: self.catalog.clone(),
+            metrics: self.metrics.clone(),
+        }
+    }
+}
+
 #[expect(unused_variables)]
 pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;

From 96c6caff7cf45cae16422002c52ac61b4a9044ec Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 13 Dec 2023 11:18:06 +0800
Subject: [PATCH 182/261] refactor: introduce inflight item into catalog,
 refactor coding (#229)

---
 foyer-common/src/code.rs                      | 128 +++++++++++++++---
 foyer-common/src/lib.rs                       |   2 +-
 foyer-storage-bench/src/main.rs               |  16 +--
 foyer-storage/src/admission/mod.rs            |  10 +-
 foyer-storage/src/admission/rated_ticket.rs   |   4 +-
 foyer-storage/src/buffer.rs                   | 120 ++++++++--------
 foyer-storage/src/catalog.rs                  |  50 ++++---
 foyer-storage/src/error.rs                    |  15 +-
 foyer-storage/src/flusher.rs                  |  70 ++++++----
 foyer-storage/src/generic.rs                  |  24 ++--
 foyer-storage/src/lib.rs                      |   2 +
 foyer-storage/src/reinsertion/exist.rs        |   4 +-
 foyer-storage/src/reinsertion/mod.rs          |  10 +-
 foyer-storage/src/reinsertion/rated_ticket.rs |   5 +-
 14 files changed, 307 insertions(+), 153 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 194b548e..e040bf3b 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -88,7 +88,7 @@ trait BufMutExt: BufMut {
 
 impl<T: BufMut> BufMutExt for T {}
 
-pub trait Cursor: Send + Sync + 'static + std::io::Read {
+pub trait Cursor: Send + Sync + 'static + std::io::Read + std::fmt::Debug {
     type T: Send + Sync + 'static;
 
     fn inner(&self) -> &Self::T;
@@ -193,11 +193,21 @@ macro_rules! def_cursor {
     ($( { $type:ty, $id:ident }, )*) => {
         paste! {
             $(
+                #[derive(Debug)]
                 pub struct [<PrimitiveCursor $id>] {
                     inner: $type,
                     pos: u8,
                 }
 
+                impl [<PrimitiveCursor $id>] {
+                    pub fn new(inner: $type) -> Self {
+                        Self {
+                            inner,
+                            pos: 0,
+                        }
+                    }
+                }
+
                 impl std::io::Read for [<PrimitiveCursor $id>] {
                     fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
                         let slice = self.inner.to_be_bytes();
@@ -239,22 +249,17 @@ macro_rules! impl_key {
                         std::mem::size_of::<$type>()
                     }
 
-
                     fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
                         buf.[< put_ $type>](*self);
                         Ok(())
                     }
 
-
                     fn read(mut buf: &[u8]) -> CodingResult<Self> {
                         Ok(buf.[< get_ $type>]())
                     }
 
                     fn into_cursor(self) -> Self::Cursor {
-                        [<PrimitiveCursor $id>] {
-                            inner: self,
-                            pos: 0,
-                        }
+                        [<PrimitiveCursor $id>]::new(self)
                     }
                 }
             )*
@@ -273,22 +278,17 @@ macro_rules! impl_value {
                         std::mem::size_of::<$type>()
                     }
 
-
                     fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
                         buf.[< put_ $type>](*self);
                         Ok(())
                     }
 
-
                     fn read(mut buf: &[u8]) -> CodingResult<Self> {
                         Ok(buf.[< get_ $type>]())
                     }
 
                     fn into_cursor(self) -> Self::Cursor {
-                        [<PrimitiveCursor $id>] {
-                            inner: self,
-                            pos: 0,
-                        }
+                        [<PrimitiveCursor $id>]::new(self)
                     }
                 }
             )*
@@ -301,6 +301,8 @@ for_all_primitives! { impl_key }
 for_all_primitives! { impl_value }
 
 impl Key for Vec<u8> {
+    type Cursor = std::io::Cursor<Vec<u8>>;
+
     fn weight(&self) -> usize {
         self.len()
     }
@@ -317,6 +319,10 @@ impl Key for Vec<u8> {
     fn read(buf: &[u8]) -> CodingResult<Self> {
         Ok(buf.to_vec())
     }
+
+    fn into_cursor(self) -> Self::Cursor {
+        std::io::Cursor::new(self)
+    }
 }
 
 impl Value for Vec<u8> {
@@ -360,6 +366,95 @@ impl Cursor for std::io::Cursor<Vec<u8>> {
     }
 }
 
+impl Key for std::sync::Arc<Vec<u8>> {
+    type Cursor = ArcVecU8Cursor;
+
+    fn weight(&self) -> usize {
+        self.len()
+    }
+
+    fn serialized_len(&self) -> usize {
+        self.len()
+    }
+
+    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
+        buf.put_slice(self);
+        Ok(())
+    }
+
+    fn read(buf: &[u8]) -> CodingResult<Self> {
+        Ok(std::sync::Arc::new(buf.to_vec()))
+    }
+
+    fn into_cursor(self) -> Self::Cursor {
+        ArcVecU8Cursor::new(self)
+    }
+}
+
+impl Value for std::sync::Arc<Vec<u8>> {
+    type Cursor = ArcVecU8Cursor;
+
+    fn weight(&self) -> usize {
+        self.len()
+    }
+
+    fn serialized_len(&self) -> usize {
+        self.len()
+    }
+
+    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
+        buf.put_slice(self);
+        Ok(())
+    }
+
+    fn read(buf: &[u8]) -> CodingResult<Self> {
+        Ok(std::sync::Arc::new(buf.to_vec()))
+    }
+
+    fn into_cursor(self) -> Self::Cursor {
+        ArcVecU8Cursor::new(self)
+    }
+}
+
+#[derive(Debug)]
+pub struct ArcVecU8Cursor {
+    inner: std::sync::Arc<Vec<u8>>,
+    pos: usize,
+}
+
+impl ArcVecU8Cursor {
+    pub fn new(inner: std::sync::Arc<Vec<u8>>) -> Self {
+        Self { inner, pos: 0 }
+    }
+}
+
+impl std::io::Read for ArcVecU8Cursor {
+    fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
+        let slice = self.inner.as_ref().as_slice();
+        let len = std::cmp::min(slice.len() - self.pos, buf.len());
+        buf.put_slice(&slice[self.pos..self.pos + len]);
+        self.pos -= len;
+        Ok(len)
+    }
+}
+
+impl Cursor for ArcVecU8Cursor {
+    type T = std::sync::Arc<Vec<u8>>;
+
+    fn inner(&self) -> &Self::T {
+        &self.inner
+    }
+
+    fn into_inner(self) -> Self::T {
+        self.inner
+    }
+
+    fn len(&self) -> usize {
+        self.inner.len()
+    }
+}
+
+#[derive(Debug)]
 pub struct PrimitiveCursorVoid;
 
 impl std::io::Read for PrimitiveCursorVoid {
@@ -424,15 +519,16 @@ impl Value for () {
     }
 }
 
-pub struct UnimplementedCursor<T: Send + Sync + 'static>(PhantomData<T>);
+#[derive(Debug)]
+pub struct UnimplementedCursor<T: Send + Sync + 'static + std::fmt::Debug>(PhantomData<T>);
 
-impl<T: Send + Sync + 'static> std::io::Read for UnimplementedCursor<T> {
+impl<T: Send + Sync + 'static + std::fmt::Debug> std::io::Read for UnimplementedCursor<T> {
     fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
         unimplemented!()
     }
 }
 
-impl<T: Send + Sync + 'static> Cursor for UnimplementedCursor<T> {
+impl<T: Send + Sync + 'static + std::fmt::Debug> Cursor for UnimplementedCursor<T> {
     type T = T;
 
     fn inner(&self) -> &Self::T {
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 1d5c401f..899032c1 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -16,8 +16,8 @@
 #![feature(lint_reasons)]
 #![feature(bound_map)]
 #![feature(associated_type_defaults)]
-#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 #![feature(cfg_match)]
+#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 
 pub mod batch;
 pub mod bits;
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index d48df812..5ccd3895 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -277,7 +277,7 @@ where
 }
 
 #[derive(Debug)]
-pub enum BenchStore<K = u64, V = Vec<u8>>
+pub enum BenchStore<K = u64, V = Arc<Vec<u8>>>
 where
     K: Key,
     V: Value,
@@ -503,8 +503,8 @@ async fn main() {
         io_size: args.io_size,
     };
 
-    let mut admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
-    let mut reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> = vec![];
+    let mut admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Arc<Vec<u8>>>>> = vec![];
+    let mut reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Arc<Vec<u8>>>>> = vec![];
     if args.ticket_insert_rate_limit > 0 {
         let rt = RatedTicketAdmissionPolicy::new(args.ticket_insert_rate_limit * 1024 * 1024);
         admissions.push(Arc::new(rt));
@@ -612,7 +612,7 @@ async fn main() {
 
 async fn bench(
     args: Args,
-    store: impl Storage<Key = u64, Value = Vec<u8>>,
+    store: impl Storage<Key = u64, Value = Arc<Vec<u8>>>,
     metrics: Metrics,
     stop_tx: broadcast::Sender<()>,
 ) {
@@ -664,7 +664,7 @@ async fn write(
     entry_size_range: Range<usize>,
     rate: Option<f64>,
     index: Arc<AtomicU64>,
-    store: impl Storage<Key = u64, Value = Vec<u8>>,
+    store: impl Storage<Key = u64, Value = Arc<Vec<u8>>>,
     time: u64,
     metrics: Metrics,
     mut stop: broadcast::Receiver<()>,
@@ -685,7 +685,7 @@ async fn write(
         let idx = index.fetch_add(1, Ordering::Relaxed);
         // TODO(MrCroxx): Use random content?
         let entry_size = OsRng.gen_range(entry_size_range.clone());
-        let data = text(idx as usize, entry_size);
+        let data = Arc::new(text(idx as usize, entry_size));
         if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
             tokio::time::sleep(wait).await;
         }
@@ -709,7 +709,7 @@ async fn write(
 async fn read(
     rate: Option<f64>,
     index: Arc<AtomicU64>,
-    store: impl Storage<Key = u64, Value = Vec<u8>>,
+    store: impl Storage<Key = u64, Value = Arc<Vec<u8>>>,
     time: u64,
     metrics: Metrics,
     mut stop: broadcast::Receiver<()>,
@@ -739,7 +739,7 @@ async fn read(
 
         if let Some(buf) = res {
             let entry_size = buf.len();
-            assert_eq!(text(idx as usize, entry_size), buf);
+            assert_eq!(&text(idx as usize, entry_size), buf.as_ref());
             if let Err(e) = metrics.get_hit_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);
             }
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 8a90724a..e966d6b4 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -19,17 +19,19 @@ use foyer_common::code::{Key, Value};
 use crate::{catalog::Catalog, metrics::Metrics};
 
 #[derive(Debug)]
-pub struct AdmissionContext<K>
+pub struct AdmissionContext<K, V>
 where
     K: Key,
+    V: Value,
 {
-    pub catalog: Arc<Catalog<K>>,
+    pub catalog: Arc<Catalog<K, V>>,
     pub metrics: Arc<Metrics>,
 }
 
-impl<K> Clone for AdmissionContext<K>
+impl<K, V> Clone for AdmissionContext<K, V>
 where
     K: Key,
+    V: Value,
 {
     fn clone(&self) -> Self {
         Self {
@@ -44,7 +46,7 @@ pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, context: AdmissionContext<Self::Key>) {}
+    fn init(&self, context: AdmissionContext<Self::Key, Self::Value>) {}
 
     fn judge(&self, key: &Self::Key, weight: usize) -> bool;
 
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
index 2dbab8d3..cc6318af 100644
--- a/foyer-storage/src/admission/rated_ticket.rs
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -38,7 +38,7 @@ where
 
     last: AtomicUsize,
 
-    context: OnceLock<AdmissionContext<K>>,
+    context: OnceLock<AdmissionContext<K, V>>,
 
     _marker: PhantomData<(K, V)>,
 }
@@ -67,7 +67,7 @@ where
 
     type Value = V;
 
-    fn init(&self, context: AdmissionContext<Self::Key>) {
+    fn init(&self, context: AdmissionContext<Self::Key, Self::Value>) {
         self.context.set(context).unwrap();
     }
 
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 5fcc96e0..82d9d088 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -12,9 +12,11 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::{fmt::Debug, marker::PhantomData};
+
 use foyer_common::{
     bits::{align_up, is_aligned},
-    code::Key,
+    code::{Key, Value},
 };
 
 use crate::{
@@ -26,28 +28,37 @@ use crate::{
 };
 
 #[derive(thiserror::Error, Debug)]
-pub enum BufferError {
-    #[error(transparent)]
+pub enum BufferError<R>
+where
+    R: Send + Sync + 'static + Debug,
+{
+    #[error("need rotate and retry {0}")]
+    NeedRotate(Box<R>),
+    #[error("device error: {0}")]
     Device(#[from] DeviceError),
-    #[error("")]
-    NotEnough { entry: Entry },
     #[error("other error: {0}")]
     Other(#[from] anyhow::Error),
 }
 
-pub type BufferResult<T> = std::result::Result<T, BufferError>;
+pub type BufferResult<T, R> = core::result::Result<T, BufferError<R>>;
 
 #[derive(Debug)]
-pub struct PositionedEntry {
-    pub entry: Entry,
+pub struct PositionedEntry<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub entry: Entry<K, V>,
     pub region: RegionId,
     pub offset: usize,
     pub len: usize,
 }
 
 #[derive(Debug)]
-pub struct FlushBuffer<D>
+pub struct FlushBuffer<K, V, D>
 where
+    K: Key,
+    V: Value,
     D: Device,
 {
     // TODO(MrCroxx): optimize buffer allocation
@@ -61,7 +72,7 @@ where
     offset: usize,
 
     /// entries in io buffer waiting for flush
-    entries: Vec<PositionedEntry>,
+    entries: Vec<PositionedEntry<K, V>>,
 
     // underlying device
     device: D,
@@ -69,8 +80,10 @@ where
     default_buffer_capacity: usize,
 }
 
-impl<D> FlushBuffer<D>
+impl<K, V, D> FlushBuffer<K, V, D>
 where
+    K: Key,
+    V: Value,
     D: Device,
 {
     pub fn new(device: D, default_buffer_capacity: usize) -> Self {
@@ -103,7 +116,10 @@ where
     /// Flush io buffer if necessary, and reset io buffer to a new region.
     ///
     /// Returns fully flushed entries.
-    pub async fn rotate(&mut self, region: RegionId) -> BufferResult<Vec<PositionedEntry>> {
+    pub async fn rotate(
+        &mut self,
+        region: RegionId,
+    ) -> BufferResult<Vec<PositionedEntry<K, V>>, Entry<K, V>> {
         let entries = self.flush().await?;
         debug_assert!(self.buffer.is_empty());
         self.region = Some(region);
@@ -126,7 +142,7 @@ where
     /// The io buffer will be cleared after flush.
     ///
     /// Returns fully flushed entries.
-    pub async fn flush(&mut self) -> BufferResult<Vec<PositionedEntry>> {
+    pub async fn flush(&mut self) -> BufferResult<Vec<PositionedEntry<K, V>>, Entry<K, V>> {
         let Some(region) = self.region else {
             debug_assert!(self.entries.is_empty());
             return Ok(vec![]);
@@ -166,28 +182,27 @@ where
     /// # Format
     ///
     /// | header | value (compressed) | key | <padding> |
-    pub async fn write<K: Key>(
+    pub async fn write(
         &mut self,
         Entry {
             key,
             sequence,
             compression,
             view,
-        }: Entry,
-    ) -> BufferResult<Vec<PositionedEntry>> {
+            _marker,
+        }: Entry<K, V>,
+    ) -> BufferResult<Vec<PositionedEntry<K, V>>, Entry<K, V>> {
         if self.region.is_none() {
-            return Err(BufferError::NotEnough {
-                entry: Entry {
-                    key,
-                    sequence,
-                    compression,
-                    view,
-                },
-            });
+            return Err(BufferError::NeedRotate(Box::new(Entry {
+                key,
+                sequence,
+                compression,
+                view,
+                _marker: PhantomData,
+            })));
         }
 
         // reserve underlying vec capacity
-        let key = key.downcast::<K>().unwrap();
         let uncompressed = align_up(
             self.device.align(),
             EntryHeader::serialized_len() + key.serialized_len() + view.len(),
@@ -202,14 +217,13 @@ where
 
         if compression == Compression::None && self.remaining() < uncompressed {
             // early return if remaining size is not enough
-            return Err(BufferError::NotEnough {
-                entry: Entry {
-                    key,
-                    sequence,
-                    compression,
-                    view,
-                },
-            });
+            return Err(BufferError::NeedRotate(Box::new(Entry {
+                key,
+                sequence,
+                compression,
+                view,
+                _marker: PhantomData,
+            })));
         }
 
         let mut cursor = self.buffer.len();
@@ -263,14 +277,13 @@ where
         // 2. if size exceeds region limit, rollback write and return
         if self.offset + self.buffer.len() > self.device.region_size() {
             unsafe { self.buffer.set_len(old) };
-            return Err(BufferError::NotEnough {
-                entry: Entry {
-                    key,
-                    sequence,
-                    compression,
-                    view,
-                },
-            });
+            return Err(BufferError::NeedRotate(Box::new(Entry {
+                key,
+                sequence,
+                compression,
+                view,
+                _marker: PhantomData,
+            })));
         }
 
         // 3. align buffer size
@@ -284,6 +297,7 @@ where
                 sequence,
                 compression,
                 view,
+                _marker: PhantomData,
             },
             region: self.region.unwrap(),
             offset: self.offset + old,
@@ -314,13 +328,13 @@ mod tests {
         ring::{RingBuffer, RingBufferView},
     };
 
-    fn ent(view: RingBufferView) -> Entry {
-        let key = Arc::new(());
+    fn ent(view: RingBufferView) -> Entry<(), ()> {
         Entry {
-            key,
+            key: (),
             view,
             compression: Compression::None,
             sequence: 0,
+            _marker: PhantomData,
         }
     }
 
@@ -355,9 +369,9 @@ mod tests {
             let entry = ent(view);
             assert_eq!(ring.continuum(), 0);
 
-            let res = buffer.write::<()>(entry).await;
+            let res = buffer.write(entry).await;
             let entry = match res {
-                Err(BufferError::NotEnough { entry }) => entry,
+                Err(BufferError::NeedRotate(entry)) => Box::into_inner(entry),
                 _ => panic!("should be not enough error"),
             };
 
@@ -365,16 +379,16 @@ mod tests {
             assert!(entries.is_empty());
 
             // 4 ~ 12 KiB
-            let entries = buffer.write::<()>(entry.clone()).await.unwrap();
+            let entries = buffer.write(entry.clone()).await.unwrap();
             assert!(entries.is_empty());
             // 12 ~ 20 KiB
-            let entries = buffer.write::<()>(entry.clone()).await.unwrap();
+            let entries = buffer.write(entry.clone()).await.unwrap();
             assert_eq!(entries.len(), 2);
             assert_eq!(entries[0].offset, 4 * 1024);
             assert_eq!(entries[1].offset, 12 * 1024);
 
             // 20 ~ 28 KiB
-            let entries = buffer.write::<()>(entry.clone()).await.unwrap();
+            let entries = buffer.write(entry.clone()).await.unwrap();
             assert!(entries.is_empty());
             let entries = buffer.flush().await.unwrap();
             assert_eq!(entries.len(), 1);
@@ -411,9 +425,9 @@ mod tests {
             };
             let entry = ent(view);
 
-            let res = buffer.write::<()>(entry).await;
+            let res = buffer.write(entry).await;
             let entry = match res {
-                Err(BufferError::NotEnough { entry }) => entry,
+                Err(BufferError::NeedRotate(entry)) => Box::into_inner(entry),
                 _ => panic!("should be not enough error"),
             };
 
@@ -421,7 +435,7 @@ mod tests {
             assert!(entries.is_empty());
 
             // 4 ~ 60 KiB
-            let entries = buffer.write::<()>(entry).await.unwrap();
+            let entries = buffer.write(entry).await.unwrap();
             assert_eq!(entries.len(), 1);
             assert_eq!(entries[0].offset, 4 * 1024);
 
@@ -434,7 +448,7 @@ mod tests {
             let entry = ent(view);
 
             // 60 ~ 64 KiB
-            let entries = buffer.write::<()>(entry).await.unwrap();
+            let entries = buffer.write(entry).await.unwrap();
             assert_eq!(entries.len(), 1);
             assert_eq!(entries[0].offset, 60 * 1024);
 
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index b88d7e5e..de37d470 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -19,13 +19,12 @@ use std::{
     time::Instant,
 };
 
-use foyer_common::code::Key;
+use foyer_common::code::{Key, Value};
 use itertools::Itertools;
 use parking_lot::{Mutex, RwLock};
 use twox_hash::XxHash64;
 
 use crate::{
-    device::BufferAllocator,
     metrics::Metrics,
     region::{RegionId, RegionView},
     ring::RingBufferView,
@@ -34,21 +33,34 @@ use crate::{
 pub type Sequence = u64;
 
 #[derive(Debug, Clone)]
-pub enum Index {
+pub enum Index<K, V>
+where
+    K: Key,
+    V: Value,
+{
     RingBuffer { view: RingBufferView },
+    Inflight { key: K, value: V },
     Region { view: RegionView },
 }
 
 #[derive(Debug, Clone)]
-pub struct Item {
+pub struct Item<K, V>
+where
+    K: Key,
+    V: Value,
+{
     sequence: Sequence,
-    index: Index,
+    index: Index<K, V>,
 
     inserted: Option<Instant>,
 }
 
-impl Item {
-    pub fn new(sequence: Sequence, index: Index) -> Self {
+impl<K, V> Item<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub fn new(sequence: Sequence, index: Index<K, V>) -> Self {
         Self {
             sequence,
             index,
@@ -60,35 +72,37 @@ impl Item {
         &self.sequence
     }
 
-    pub fn index(&self) -> &Index {
+    pub fn index(&self) -> &Index<K, V> {
         &self.index
     }
 
-    pub fn consume(self) -> (Sequence, Index) {
+    pub fn consume(self) -> (Sequence, Index<K, V>) {
         (self.sequence, self.index)
     }
 }
 
 #[derive(Debug)]
-pub struct Catalog<K>
+pub struct Catalog<K, V>
 where
     K: Key,
+    V: Value,
 {
     /// `items` sharding bits.
     bits: usize,
 
     /// Sharded by key hash.
-    items: Vec<RwLock<BTreeMap<Arc<K>, Item>>>,
+    items: Vec<RwLock<BTreeMap<K, Item<K, V>>>>,
 
     /// Sharded by region id.
-    regions: Vec<Mutex<BTreeMap<Arc<K>, u64>>>,
+    regions: Vec<Mutex<BTreeMap<K, u64>>>,
 
     metrics: Arc<Metrics>,
 }
 
-impl<K> Catalog<K>
+impl<K, V> Catalog<K, V>
 where
     K: Key,
+    V: Value,
 {
     pub fn new(regions: usize, bits: usize, metrics: Arc<Metrics>) -> Self {
         let infos = (0..1 << bits)
@@ -106,7 +120,7 @@ where
         }
     }
 
-    pub fn insert(&self, key: Arc<K>, mut item: Item) {
+    pub fn insert(&self, key: K, mut item: Item<K, V>) {
         // TODO(MrCroxx): compare sequence.
 
         if let Index::Region { view } = &item.index {
@@ -127,21 +141,21 @@ where
         }
     }
 
-    pub fn lookup(&self, key: &K) -> Option<Item> {
+    pub fn lookup(&self, key: &K) -> Option<Item<K, V>> {
         let shard = self.shard(key);
         self.items[shard].read().get(key).cloned()
     }
 
-    pub fn remove(&self, key: &K) -> Option<Item> {
+    pub fn remove(&self, key: &K) -> Option<Item<K, V>> {
         let shard = self.shard(key);
-        let info: Option<Item> = self.items[shard].write().remove(key);
+        let info: Option<Item<K, V>> = self.items[shard].write().remove(key);
         if let Some(info) = &info && let Index::Region { view} = &info.index {
             self.regions[*view.id() as usize].lock().remove(key);
         }
         info
     }
 
-    pub fn take_region(&self, region: &RegionId) -> Vec<(Arc<K>, Item)> {
+    pub fn take_region(&self, region: &RegionId) -> Vec<(K, Item<K, V>)> {
         let mut keys = BTreeMap::new();
         std::mem::swap(&mut *self.regions[*region as usize].lock(), &mut keys);
 
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index b484dda1..e18363a0 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -31,7 +31,7 @@ pub enum ErrorKind {
     #[error("device error: {0}")]
     Device(#[from] DeviceError),
     #[error("buffer error: {0}")]
-    Buffer(#[from] BufferError),
+    Buffer(anyhow::Error),
     #[error("other error: {0}")]
     Other(#[from] anyhow::Error),
 }
@@ -48,9 +48,16 @@ impl From<DeviceError> for Error {
     }
 }
 
-impl From<BufferError> for Error {
-    fn from(value: BufferError) -> Self {
-        value.into()
+impl<R> From<BufferError<R>> for Error
+where
+    R: Send + Sync + 'static + std::fmt::Debug,
+{
+    fn from(value: BufferError<R>) -> Self {
+        match value {
+            BufferError::NeedRotate(_) => panic!("BufferError::NeedRotate should not be raised!"),
+            BufferError::Device(e) => From::from(e),
+            BufferError::Other(e) => From::from(e),
+        }
     }
 }
 
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 4741c75e..281ccc40 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -12,9 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{any::Any, fmt::Debug, sync::Arc};
+use std::{fmt::Debug, marker::PhantomData, sync::Arc};
 
-use foyer_common::code::Key;
+use foyer_common::code::{Key, Value};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
 use tokio::sync::{broadcast, mpsc};
 use tracing::Instrument;
@@ -24,27 +24,32 @@ use crate::{
     catalog::{Catalog, Index, Item, Sequence},
     compress::Compression,
     device::Device,
-    error::{Error, Result},
+    error::Result,
     metrics::Metrics,
     region_manager::{RegionEpItemAdapter, RegionManager},
     ring::RingBufferView,
 };
 
-pub struct Entry {
-    /// # Safety
-    ///
-    /// `key` must be `Arc<K> where K = Flusher<K>`.
-    ///
-    /// Use `dyn Any` here to avoid contagious generic type.
-    pub key: Arc<dyn Any + Send + Sync>,
+pub struct Entry<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    pub key: K,
     pub sequence: Sequence,
     pub compression: Compression,
 
     /// Hold a view of referenced buffer, for lookup and prevent from releasing.
     pub view: RingBufferView,
+
+    pub _marker: PhantomData<V>,
 }
 
-impl Debug for Entry {
+impl<K, V> Debug for Entry<K, V>
+where
+    K: Key,
+    V: Value,
+{
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Entry")
             .field("sequence", &self.sequence)
@@ -53,41 +58,48 @@ impl Debug for Entry {
     }
 }
 
-impl Clone for Entry {
+impl<K, V> Clone for Entry<K, V>
+where
+    K: Key,
+    V: Value,
+{
     fn clone(&self) -> Self {
         Self {
-            key: Arc::clone(&self.key),
+            key: self.key.clone(),
             view: self.view.clone(),
             sequence: self.sequence,
             compression: self.compression,
+            _marker: PhantomData,
         }
     }
 }
 
 #[derive(Debug)]
-pub struct Flusher<K, D, EP, EL>
+pub struct Flusher<K, V, D, EP, EL>
 where
     K: Key,
+    V: Value,
     D: Device,
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
 {
     region_manager: Arc<RegionManager<D, EP, EL>>,
 
-    catalog: Arc<Catalog<K>>,
+    catalog: Arc<Catalog<K, V>>,
 
-    buffer: FlushBuffer<D>,
+    buffer: FlushBuffer<K, V, D>,
 
-    entry_rx: mpsc::UnboundedReceiver<Entry>,
+    entry_rx: mpsc::UnboundedReceiver<Entry<K, V>>,
 
     metrics: Arc<Metrics>,
 
     stop_rx: broadcast::Receiver<()>,
 }
 
-impl<K, D, EP, EL> Flusher<K, D, EP, EL>
+impl<K, V, D, EP, EL> Flusher<K, V, D, EP, EL>
 where
     K: Key,
+    V: Value,
     D: Device,
     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
     EL: Link,
@@ -95,9 +107,9 @@ where
     pub fn new(
         default_buffer_capacity: usize,
         region_manager: Arc<RegionManager<D, EP, EL>>,
-        catalog: Arc<Catalog<K>>,
+        catalog: Arc<Catalog<K, V>>,
         device: D,
-        entry_rx: mpsc::UnboundedReceiver<Entry>,
+        entry_rx: mpsc::UnboundedReceiver<Entry<K, V>>,
         metrics: Arc<Metrics>,
         stop_rx: broadcast::Receiver<()>,
     ) -> Self {
@@ -133,16 +145,15 @@ where
         }
     }
 
-    async fn handle(&mut self, entry: Entry) -> Result<()> {
+    async fn handle(&mut self, entry: Entry<K, V>) -> Result<()> {
         let timer = self.metrics.inner_op_duration_flusher_handle.start_timer();
 
         let old_region = self.buffer.region();
 
-        let entry = match self.buffer.write::<K>(entry).await {
-            Err(BufferError::NotEnough { entry }) => entry,
-
+        let entry = match self.buffer.write(entry).await {
+            Err(BufferError::NeedRotate(entry)) => Box::into_inner(entry),
             Ok(entries) => return self.update_catalog(entries).await,
-            Err(e) => return Err(Error::from(e)),
+            Err(e) => return Err(e.into()),
         };
 
         // current region is full, rotate flush buffer region and retry
@@ -175,7 +186,11 @@ where
         );
 
         // 3. retry write
-        let entries = self.buffer.write::<K>(entry).await?;
+        let entries = match self.buffer.write(entry).await {
+            Err(BufferError::NeedRotate(_)) => unreachable!(),
+            result => result?,
+        };
+
         self.update_catalog(entries).await?;
 
         drop(timer);
@@ -183,7 +198,7 @@ where
     }
 
     #[tracing::instrument(skip(self))]
-    async fn update_catalog(&self, entries: Vec<PositionedEntry>) -> Result<()> {
+    async fn update_catalog(&self, entries: Vec<PositionedEntry<K, V>>) -> Result<()> {
         // record fully flushed bytes by the way
         let mut bytes = 0;
 
@@ -196,7 +211,6 @@ where
         } in entries
         {
             bytes += len;
-            let key = key.downcast::<K>().unwrap();
             let index = Index::Region {
                 view: self
                     .region_manager
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 625efa29..c3f0e14a 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -198,7 +198,7 @@ where
     EL: Link,
 {
     sequence: AtomicU64,
-    catalog: Arc<Catalog<K>>,
+    catalog: Arc<Catalog<K, V>>,
 
     region_manager: Arc<RegionManager<D, EP, EL>>,
     ring: Arc<RingBuffer<D::IoBufferAllocator>>,
@@ -208,7 +208,7 @@ where
     admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
     reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
 
-    flusher_entry_txs: Vec<mpsc::UnboundedSender<Entry>>,
+    flusher_entry_txs: Vec<mpsc::UnboundedSender<Entry<K, V>>>,
     flusher_handles: Mutex<Vec<JoinHandle<()>>>,
     flushers_stop_tx: broadcast::Sender<()>,
 
@@ -261,9 +261,10 @@ where
         let flusher_stop_rxs = (0..config.flushers)
             .map(|_| flushers_stop_tx.subscribe())
             .collect_vec();
+        #[expect(clippy::type_complexity)]
         let (flusher_entry_txs, flusher_entry_rxs): (
-            Vec<mpsc::UnboundedSender<Entry>>,
-            Vec<mpsc::UnboundedReceiver<Entry>>,
+            Vec<mpsc::UnboundedSender<Entry<K, V>>>,
+            Vec<mpsc::UnboundedReceiver<Entry<K, V>>>,
         ) = (0..config.flushers)
             .map(|_| mpsc::unbounded_channel())
             .unzip();
@@ -421,6 +422,10 @@ where
 
                 Ok(Some(value))
             }
+            crate::catalog::Index::Inflight { key: _, value } => {
+                let value = value.clone();
+                Ok(Some(value))
+            }
             // read from region
             crate::catalog::Index::Region { view } => {
                 let region = view.id();
@@ -485,7 +490,7 @@ where
         Ok(())
     }
 
-    pub(crate) fn catalog(&self) -> &Arc<Catalog<K>> {
+    pub(crate) fn catalog(&self) -> &Arc<Catalog<K, V>> {
         &self.inner.catalog
     }
 
@@ -544,14 +549,14 @@ where
     async fn recover_region(
         region_id: RegionId,
         region_manager: Arc<RegionManager<D, EP, EL>>,
-        catalog: Arc<Catalog<K>>,
+        catalog: Arc<Catalog<K, V>>,
     ) -> Result<Option<Sequence>> {
         let region = region_manager.region(&region_id).clone();
         let mut sequence = 0;
         let res = if let Some(mut iter) = RegionEntryIter::<K, V, D>::open(region).await? {
             while let Some((key, item)) = iter.next().await? {
                 sequence = std::cmp::max(sequence, *item.sequence());
-                catalog.insert(Arc::new(key), item);
+                catalog.insert(key, item);
             }
             region_manager.eviction_push(region_id);
             Some(sequence)
@@ -617,8 +622,6 @@ where
         view.shrink_to(value.serialized_len());
         let view = view.freeze();
 
-        let key = Arc::new(key);
-
         self.inner.catalog.insert(
             key.clone(),
             Item::new(sequence, Index::RingBuffer { view: view.clone() }),
@@ -631,6 +634,7 @@ where
                 key,
                 view,
                 compression: writer.compression,
+                _marker: PhantomData,
             })
             .unwrap();
 
@@ -944,7 +948,7 @@ where
         }))
     }
 
-    pub async fn next(&mut self) -> Result<Option<(K, Item)>> {
+    pub async fn next(&mut self) -> Result<Option<(K, Item<K, V>)>> {
         let region_size = self.region.device().region_size();
         let align = self.region.device().align();
 
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 762dd528..b2bcac1e 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -21,6 +21,8 @@
 #![feature(lazy_cell)]
 #![feature(lint_reasons)]
 #![feature(associated_type_defaults)]
+#![feature(box_into_inner)]
+#![feature(try_trait_v2)]
 
 pub mod admission;
 pub mod buffer;
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index 7af2ccbc..69718000 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -28,7 +28,7 @@ where
     K: Key,
     V: Value,
 {
-    catalog: OnceLock<Arc<Catalog<K>>>,
+    catalog: OnceLock<Arc<Catalog<K, V>>>,
     _marker: PhantomData<V>,
 }
 
@@ -54,7 +54,7 @@ where
 
     type Value = V;
 
-    fn init(&self, context: ReinsertionContext<Self::Key>) {
+    fn init(&self, context: ReinsertionContext<Self::Key, Self::Value>) {
         self.catalog.get_or_init(|| context.catalog.clone());
     }
 
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index afb294e2..3fe93f93 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -19,17 +19,19 @@ use foyer_common::code::{Key, Value};
 use crate::{catalog::Catalog, metrics::Metrics};
 
 #[derive(Debug)]
-pub struct ReinsertionContext<K>
+pub struct ReinsertionContext<K, V>
 where
     K: Key,
+    V: Value,
 {
-    pub catalog: Arc<Catalog<K>>,
+    pub catalog: Arc<Catalog<K, V>>,
     pub metrics: Arc<Metrics>,
 }
 
-impl<K> Clone for ReinsertionContext<K>
+impl<K, V> Clone for ReinsertionContext<K, V>
 where
     K: Key,
+    V: Value,
 {
     fn clone(&self) -> Self {
         Self {
@@ -44,7 +46,7 @@ pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, context: ReinsertionContext<Self::Key>) {}
+    fn init(&self, context: ReinsertionContext<Self::Key, Self::Value>) {}
 
     fn judge(&self, key: &Self::Key, weight: usize) -> bool;
 
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
index afebaf02..d96ef806 100644
--- a/foyer-storage/src/reinsertion/rated_ticket.rs
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -38,7 +38,7 @@ where
 
     last: AtomicUsize,
 
-    context: OnceLock<ReinsertionContext<K>>,
+    context: OnceLock<ReinsertionContext<K, V>>,
 
     _marker: PhantomData<(K, V)>,
 }
@@ -64,10 +64,9 @@ where
     V: Value,
 {
     type Key = K;
-
     type Value = V;
 
-    fn init(&self, context: super::ReinsertionContext<Self::Key>) {
+    fn init(&self, context: super::ReinsertionContext<Self::Key, Self::Value>) {
         self.context.set(context).unwrap();
     }
 

From 1942045f2580022911162a5c8e6b89f68479695a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 15 Dec 2023 12:07:29 +0800
Subject: [PATCH 183/261] refactor: use key/value cursor for serialization,
 retire ring buffer (#230)

---
 .github/template/template.yml       |   4 +-
 .github/workflows/main.yml          |   4 +-
 .github/workflows/pull-request.yml  |   4 +-
 foyer-common/src/code.rs            |  52 ++---------
 foyer-storage-bench/src/main.rs     |   5 --
 foyer-storage/src/buffer.rs         | 132 +++++++++++-----------------
 foyer-storage/src/catalog.rs        |   4 +-
 foyer-storage/src/flusher.rs        |  14 +--
 foyer-storage/src/generic.rs        |  54 +++---------
 foyer-storage/src/lazy.rs           |   2 -
 foyer-storage/tests/storage_test.rs |   6 --
 11 files changed, 76 insertions(+), 205 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index fb14a4f1..3c6ecdd2 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -120,7 +120,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -149,7 +149,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a852da33..0583a71c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -127,7 +127,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -156,7 +156,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 5dfb8d47..911c379f 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -126,7 +126,7 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
-          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --time 60
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -155,7 +155,7 @@ jobs:
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index e040bf3b..86979876 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -130,10 +130,6 @@ pub trait Key:
         panic!("Method `serialized_len` must be implemented for `Key` if storage is used.")
     }
 
-    fn write(&self, buf: &mut [u8]) -> CodingResult<()> {
-        panic!("Method `write` must be implemented for `Key` if storage is used.")
-    }
-
     fn read(buf: &[u8]) -> CodingResult<Self> {
         panic!("Method `read` must be implemented for `Key` if storage is used.")
     }
@@ -159,10 +155,6 @@ pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug + Clone {
         panic!("Method `serialized_len` must be implemented for `Value` if storage is used.")
     }
 
-    fn write(&self, buf: &mut [u8]) -> CodingResult<()> {
-        panic!("Method `write` must be implemented for `Value` if storage is used.")
-    }
-
     fn read(buf: &[u8]) -> CodingResult<Self> {
         panic!("Method `read` must be implemented for `Value` if storage is used.")
     }
@@ -249,11 +241,6 @@ macro_rules! impl_key {
                         std::mem::size_of::<$type>()
                     }
 
-                    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
-                        buf.[< put_ $type>](*self);
-                        Ok(())
-                    }
-
                     fn read(mut buf: &[u8]) -> CodingResult<Self> {
                         Ok(buf.[< get_ $type>]())
                     }
@@ -278,11 +265,6 @@ macro_rules! impl_value {
                         std::mem::size_of::<$type>()
                     }
 
-                    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
-                        buf.[< put_ $type>](*self);
-                        Ok(())
-                    }
-
                     fn read(mut buf: &[u8]) -> CodingResult<Self> {
                         Ok(buf.[< get_ $type>]())
                     }
@@ -311,11 +293,6 @@ impl Key for Vec<u8> {
         self.len()
     }
 
-    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
-        buf.put_slice(self);
-        Ok(())
-    }
-
     fn read(buf: &[u8]) -> CodingResult<Self> {
         Ok(buf.to_vec())
     }
@@ -336,11 +313,6 @@ impl Value for Vec<u8> {
         self.len()
     }
 
-    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
-        buf.put_slice(self);
-        Ok(())
-    }
-
     fn read(buf: &[u8]) -> CodingResult<Self> {
         Ok(buf.to_vec())
     }
@@ -377,11 +349,6 @@ impl Key for std::sync::Arc<Vec<u8>> {
         self.len()
     }
 
-    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
-        buf.put_slice(self);
-        Ok(())
-    }
-
     fn read(buf: &[u8]) -> CodingResult<Self> {
         Ok(std::sync::Arc::new(buf.to_vec()))
     }
@@ -402,11 +369,6 @@ impl Value for std::sync::Arc<Vec<u8>> {
         self.len()
     }
 
-    fn write(&self, mut buf: &mut [u8]) -> CodingResult<()> {
-        buf.put_slice(self);
-        Ok(())
-    }
-
     fn read(buf: &[u8]) -> CodingResult<Self> {
         Ok(std::sync::Arc::new(buf.to_vec()))
     }
@@ -433,7 +395,7 @@ impl std::io::Read for ArcVecU8Cursor {
         let slice = self.inner.as_ref().as_slice();
         let len = std::cmp::min(slice.len() - self.pos, buf.len());
         buf.put_slice(&slice[self.pos..self.pos + len]);
-        self.pos -= len;
+        self.pos += len;
         Ok(len)
     }
 }
@@ -478,6 +440,8 @@ impl Cursor for PrimitiveCursorVoid {
 }
 
 impl Key for () {
+    type Cursor = PrimitiveCursorVoid;
+
     fn weight(&self) -> usize {
         0
     }
@@ -486,12 +450,12 @@ impl Key for () {
         0
     }
 
-    fn write(&self, _buf: &mut [u8]) -> CodingResult<()> {
+    fn read(_buf: &[u8]) -> CodingResult<Self> {
         Ok(())
     }
 
-    fn read(_buf: &[u8]) -> CodingResult<Self> {
-        Ok(())
+    fn into_cursor(self) -> Self::Cursor {
+        PrimitiveCursorVoid
     }
 }
 
@@ -506,10 +470,6 @@ impl Value for () {
         0
     }
 
-    fn write(&self, _buf: &mut [u8]) -> CodingResult<()> {
-        Ok(())
-    }
-
     fn read(_buf: &[u8]) -> CodingResult<Self> {
         Ok(())
     }
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 5ccd3895..5b488202 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -105,10 +105,6 @@ pub struct Args {
     #[arg(long, default_value_t = 0)]
     flusher_buffer_size: usize,
 
-    /// (MiB)
-    #[arg(long, default_value_t = 1024)]
-    ring_buffer_capacity: usize,
-
     #[arg(long, default_value_t = 4)]
     flushers: usize,
 
@@ -530,7 +526,6 @@ async fn main() {
         name: "".to_string(),
         eviction_config,
         device_config,
-        ring_buffer_capacity: args.ring_buffer_capacity * 1024 * 1024,
         catalog_bits: args.catalog_bits,
         admissions,
         reinsertions,
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 82d9d088..c191ad89 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -12,11 +12,11 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData};
+use std::fmt::Debug;
 
 use foyer_common::{
     bits::{align_up, is_aligned},
-    code::{Key, Value},
+    code::{Cursor, Key, Value},
 };
 
 use crate::{
@@ -182,62 +182,57 @@ where
     /// # Format
     ///
     /// | header | value (compressed) | key | <padding> |
+    #[expect(clippy::uninit_vec)]
     pub async fn write(
         &mut self,
         Entry {
             key,
+            value,
             sequence,
             compression,
-            view,
-            _marker,
         }: Entry<K, V>,
     ) -> BufferResult<Vec<PositionedEntry<K, V>>, Entry<K, V>> {
+        // Notify caller to rotate buffer if there is not enough space for the entry.
+        //
+        // NOTICE:
+        //
+        // Buffer remaining size is not compared here because the compressed entry size can be
+        // either larger (rarely) or smaller than the uncompressed size and it can not be determined
+        // before compression. So we first try to compress it and rollback if it exceeds region size.
+        //
+        // P.S. About rollback, see (*).
         if self.region.is_none() {
             return Err(BufferError::NeedRotate(Box::new(Entry {
                 key,
+                value,
                 sequence,
                 compression,
-                view,
-                _marker: PhantomData,
             })));
         }
 
-        // reserve underlying vec capacity
-        let uncompressed = align_up(
-            self.device.align(),
-            EntryHeader::serialized_len() + key.serialized_len() + view.len(),
-        );
-        self.buffer.reserve_exact(uncompressed);
-
-        // TODO(MrCroxx): skip if remaining is too small
-
-        // 1. try compress and write first
         let old = self.buffer.len();
         debug_assert!(is_aligned(self.device.align(), old));
 
-        if compression == Compression::None && self.remaining() < uncompressed {
-            // early return if remaining size is not enough
-            return Err(BufferError::NeedRotate(Box::new(Entry {
-                key,
-                sequence,
-                compression,
-                view,
-                _marker: PhantomData,
-            })));
-        }
-
-        let mut cursor = self.buffer.len();
+        // reserve underlying buffer to reduce reallocation
+        let uncompressed = align_up(
+            self.device.align(),
+            EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len(),
+        );
+        self.buffer.reserve(old + uncompressed);
 
+        let mut cursor = old;
         // reserve space for header
         cursor += EntryHeader::serialized_len();
         unsafe { self.buffer.set_len(cursor) };
 
+        // write value
+        let mut vcursor = value.into_cursor();
         match compression {
             Compression::None => {
-                std::io::copy(&mut &view[..], &mut self.buffer).map_err(DeviceError::from)?;
+                std::io::copy(&mut vcursor, &mut self.buffer).map_err(DeviceError::from)?;
             }
             Compression::Zstd => {
-                zstd::stream::copy_encode(&mut &view[..], &mut self.buffer, 0)
+                zstd::stream::copy_encode(&mut vcursor, &mut self.buffer, 0)
                     .map_err(DeviceError::from)?;
             }
             Compression::Lz4 => {
@@ -245,28 +240,29 @@ where
                     .checksum(lz4::ContentChecksum::NoChecksum)
                     .build(&mut self.buffer)
                     .map_err(DeviceError::from)?;
-                std::io::copy(&mut &view[..], &mut encoder).map_err(DeviceError::from)?;
+                std::io::copy(&mut vcursor, &mut encoder).map_err(DeviceError::from)?;
                 let (_w, res) = encoder.finish();
                 res.map_err(DeviceError::from)?;
             }
         }
         let compressed_value_len = self.buffer.len() - cursor;
+        cursor = self.buffer.len();
 
         // write key
-        cursor += compressed_value_len;
-        self.buffer.reserve_exact(key.serialized_len());
-        unsafe { self.buffer.set_len(cursor + key.serialized_len()) };
-        key.write(&mut self.buffer[cursor..cursor + key.serialized_len()])?;
+        let mut kcursor = key.into_cursor();
+        std::io::copy(&mut kcursor, &mut self.buffer).map_err(DeviceError::from)?;
+        let encoded_key_len = self.buffer.len() - cursor;
+        cursor = self.buffer.len();
 
         // calculate checksum
-        cursor -= compressed_value_len;
+        cursor -= compressed_value_len + encoded_key_len;
         let checksum =
-            checksum(&self.buffer[cursor..cursor + compressed_value_len + key.serialized_len()]);
+            checksum(&self.buffer[cursor..cursor + compressed_value_len + encoded_key_len]);
 
         // write entry header
         cursor -= EntryHeader::serialized_len();
         let header = EntryHeader {
-            key_len: key.serialized_len() as u32,
+            key_len: encoded_key_len as u32,
             value_len: compressed_value_len as u32,
             sequence,
             compression,
@@ -274,30 +270,33 @@ where
         };
         header.write(&mut self.buffer[cursor..cursor + EntryHeader::serialized_len()]);
 
-        // 2. if size exceeds region limit, rollback write and return
+        // (*) if size exceeds region limit, rollback write and return
         if self.offset + self.buffer.len() > self.device.region_size() {
             unsafe { self.buffer.set_len(old) };
+            let key = kcursor.into_inner();
+            let value = vcursor.into_inner();
             return Err(BufferError::NeedRotate(Box::new(Entry {
                 key,
+                value,
                 sequence,
                 compression,
-                view,
-                _marker: PhantomData,
             })));
         }
 
         // 3. align buffer size
         let target = align_up(self.device.align(), self.buffer.len());
-        self.buffer.reserve_exact(target - self.buffer.len());
+        self.buffer.reserve(target - self.buffer.len());
         unsafe { self.buffer.set_len(target) }
 
+        let key = kcursor.into_inner();
+        let value = vcursor.into_inner();
+
         self.entries.push(PositionedEntry {
             entry: Entry {
                 key,
+                value,
                 sequence,
                 compression,
-                view,
-                _marker: PhantomData,
             },
             region: self.region.unwrap(),
             offset: self.offset + old,
@@ -317,24 +316,17 @@ where
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
-    use bytes::BufMut;
     use tempfile::tempdir;
 
     use super::*;
-    use crate::{
-        device::fs::{FsDevice, FsDeviceConfig},
-        ring::{RingBuffer, RingBufferView},
-    };
+    use crate::device::fs::{FsDevice, FsDeviceConfig};
 
-    fn ent(view: RingBufferView) -> Entry<(), ()> {
+    fn ent(size: usize) -> Entry<(), Vec<u8>> {
         Entry {
             key: (),
-            view,
+            value: vec![b'x'; size],
             compression: Compression::None,
             sequence: 0,
-            _marker: PhantomData,
         }
     }
 
@@ -342,7 +334,6 @@ mod tests {
     async fn test_flush_buffer() {
         let tempdir = tempdir().unwrap();
 
-        let ring = Arc::new(RingBuffer::new(4096, 16 * 1024 * 1024));
         let device = FsDevice::open(FsDeviceConfig {
             dir: tempdir.path().into(),
             capacity: 256 * 1024,     // 256 KiB
@@ -360,14 +351,7 @@ mod tests {
         const HEADER: usize = EntryHeader::serialized_len();
 
         {
-            let view = {
-                let mut view = ring.allocate(5 * 1024 - 128, 0).await; // ~ 6 KiB
-                (&mut view[..]).put_slice(&[b'x'; 5 * 1024 - 128]);
-                view.shrink_to(5 * 1024 - 128); // ~ 5 KiB
-                view.freeze()
-            };
-            let entry = ent(view);
-            assert_eq!(ring.continuum(), 0);
+            let entry = ent(5 * 1024 - 128); // ~ 5 KiB
 
             let res = buffer.write(entry).await;
             let entry = match res {
@@ -413,17 +397,8 @@ mod tests {
             assert!(buffer.entries.is_empty());
         }
 
-        ring.advance();
-        assert_eq!(ring.continuum(), 8 * 1024);
-
         {
-            let view = {
-                let mut view = ring.allocate(55 * 1024 - 128, 1).await; // ~ 55 KiB
-                (&mut view[..]).put_slice(&[b'x'; 54 * 1024 - 128]);
-                view.shrink_to(54 * 1024 - 128); // ~ 54 KiB
-                view.freeze()
-            };
-            let entry = ent(view);
+            let entry = ent(54 * 1024 - 128); // ~ 54 KiB
 
             let res = buffer.write(entry).await;
             let entry = match res {
@@ -439,13 +414,7 @@ mod tests {
             assert_eq!(entries.len(), 1);
             assert_eq!(entries[0].offset, 4 * 1024);
 
-            let view = {
-                let mut view = ring.allocate(3 * 1024 - 128, 2).await; // ~ 3 KiB
-                (&mut view[..]).put_slice(&[b'x'; 3 * 1024 - 128]);
-                view.shrink_to(3 * 1024 - 128);
-                view.freeze()
-            };
-            let entry = ent(view);
+            let entry = ent(3 * 1024 - 128); // ~ 3 KiB
 
             // 60 ~ 64 KiB
             let entries = buffer.write(entry).await.unwrap();
@@ -466,8 +435,5 @@ mod tests {
 
             assert!(buffer.entries.is_empty());
         }
-
-        ring.advance();
-        assert_eq!(ring.continuum(), 68 * 1024);
     }
 }
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index de37d470..7dd94267 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -27,7 +27,6 @@ use twox_hash::XxHash64;
 use crate::{
     metrics::Metrics,
     region::{RegionId, RegionView},
-    ring::RingBufferView,
 };
 
 pub type Sequence = u64;
@@ -38,7 +37,6 @@ where
     K: Key,
     V: Value,
 {
-    RingBuffer { view: RingBufferView },
     Inflight { key: K, value: V },
     Region { view: RegionView },
 }
@@ -136,7 +134,7 @@ where
             item.inserted = Some(Instant::now());
             guard.insert(key.clone(), item)
         };
-        if let Some(old) = old && let Index::RingBuffer { .. } = old.index() {
+        if let Some(old) = old && let Index::Inflight { .. } = old.index() {
             self.metrics.inner_op_duration_entry_flush.observe(old.inserted.unwrap().elapsed().as_secs_f64());
         }
     }
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 281ccc40..8b71d8b5 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc};
+use std::{fmt::Debug, sync::Arc};
 
 use foyer_common::code::{Key, Value};
 use foyer_intrusive::{core::adapter::Link, eviction::EvictionPolicy};
@@ -27,7 +27,6 @@ use crate::{
     error::Result,
     metrics::Metrics,
     region_manager::{RegionEpItemAdapter, RegionManager},
-    ring::RingBufferView,
 };
 
 pub struct Entry<K, V>
@@ -36,13 +35,9 @@ where
     V: Value,
 {
     pub key: K,
+    pub value: V,
     pub sequence: Sequence,
     pub compression: Compression,
-
-    /// Hold a view of referenced buffer, for lookup and prevent from releasing.
-    pub view: RingBufferView,
-
-    pub _marker: PhantomData<V>,
 }
 
 impl<K, V> Debug for Entry<K, V>
@@ -53,7 +48,7 @@ where
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Entry")
             .field("sequence", &self.sequence)
-            .field("view", &self.view)
+            .field("compression", &self.compression)
             .finish()
     }
 }
@@ -66,10 +61,9 @@ where
     fn clone(&self) -> Self {
         Self {
             key: self.key.clone(),
-            view: self.view.clone(),
+            value: self.value.clone(),
             sequence: self.sequence,
             compression: self.compression,
-            _marker: PhantomData,
         }
     }
 }
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index c3f0e14a..9f34f64c 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -53,7 +53,6 @@ use crate::{
     region::{Region, RegionHeader, RegionId},
     region_manager::{RegionEpItemAdapter, RegionManager},
     reinsertion::{ReinsertionContext, ReinsertionPolicy},
-    ring::RingBuffer,
     storage::{Storage, StorageWriter},
 };
 
@@ -77,9 +76,6 @@ where
     /// Device configurations.
     pub device_config: D::Config,
 
-    /// `ring_buffer_capacity` will be aligned up to device align.
-    pub ring_buffer_capacity: usize,
-
     /// Catalog indices sharding bits.
     pub catalog_bits: usize,
 
@@ -121,7 +117,6 @@ where
         f.debug_struct("StoreConfig")
             .field("eviction_config", &self.eviction_config)
             .field("device_config", &self.device_config)
-            .field("ring_buffer_capacity", &self.ring_buffer_capacity)
             .field("catalog_bits", &self.catalog_bits)
             .field("admissions", &self.admissions)
             .field("reinsertions", &self.reinsertions)
@@ -147,7 +142,6 @@ where
             name: self.name.clone(),
             eviction_config: self.eviction_config.clone(),
             device_config: self.device_config.clone(),
-            ring_buffer_capacity: self.ring_buffer_capacity,
             catalog_bits: self.catalog_bits,
             admissions: self.admissions.clone(),
             reinsertions: self.reinsertions.clone(),
@@ -201,7 +195,6 @@ where
     catalog: Arc<Catalog<K, V>>,
 
     region_manager: Arc<RegionManager<D, EP, EL>>,
-    ring: Arc<RingBuffer<D::IoBufferAllocator>>,
 
     device: D,
 
@@ -238,13 +231,6 @@ where
         let device = D::open(config.device_config).await?;
         assert!(device.regions() >= config.flushers * 2);
 
-        let ring = Arc::new(RingBuffer::with_metrics_in(
-            device.align(),
-            config.ring_buffer_capacity,
-            metrics.clone(),
-            device.io_buffer_allocator().clone(),
-        ));
-
         let region_manager = Arc::new(RegionManager::new(
             device.regions(),
             config.eviction_config,
@@ -278,7 +264,6 @@ where
             sequence: AtomicU64::new(0),
             catalog: catalog.clone(),
             region_manager: region_manager.clone(),
-            ring,
             device: device.clone(),
             admissions: config.admissions,
             reinsertions: config.reinsertions,
@@ -407,23 +392,14 @@ where
         };
 
         match index {
-            crate::catalog::Index::RingBuffer { view } => {
-                let value = V::read(&view)?;
+            crate::catalog::Index::Inflight { key: _, value } => {
+                let value = value.clone();
 
                 self.inner
                     .metrics
                     .op_duration_lookup_hit
                     .observe(now.elapsed().as_secs_f64());
 
-                self.inner
-                    .metrics
-                    .op_bytes_lookup
-                    .inc_by(value.serialized_len() as u64);
-
-                Ok(Some(value))
-            }
-            crate::catalog::Index::Inflight { key: _, value } => {
-                let value = value.clone();
                 Ok(Some(value))
             }
             // read from region
@@ -603,28 +579,21 @@ where
             admission.on_insert(&key, writer.weight, judge);
         }
 
-        // only write value to ring buffer
-        let len = bits::align_up(self.inner.device.align(), value.serialized_len());
-
         // record aligned header + key + value size for metrics
         self.inner.metrics.op_bytes_insert.inc_by(bits::align_up(
             self.inner.device.align(),
             EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len(),
         ) as u64);
 
-        self.inner
-            .metrics
-            .inner_bytes_ring_buffer_remains
-            .set(self.inner.ring.remains() as i64);
-        let mut view = self.inner.ring.allocate(len, sequence).await;
-        value.write(&mut view)?;
-
-        view.shrink_to(value.serialized_len());
-        let view = view.freeze();
-
         self.inner.catalog.insert(
             key.clone(),
-            Item::new(sequence, Index::RingBuffer { view: view.clone() }),
+            Item::new(
+                sequence,
+                Index::Inflight {
+                    key: key.clone(),
+                    value: value.clone(),
+                },
+            ),
         );
 
         let flusher = sequence as usize % self.inner.flusher_entry_txs.len();
@@ -632,9 +601,8 @@ where
             .send(Entry {
                 sequence,
                 key,
-                view,
+                value,
                 compression: writer.compression,
-                _marker: PhantomData,
             })
             .unwrap();
 
@@ -1176,7 +1144,6 @@ mod tests {
                 align: 4 * KB,
                 io_size: 4 * KB,
             },
-            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions,
             reinsertions,
@@ -1227,7 +1194,6 @@ mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index c64b9db4..26c08246 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -253,7 +253,6 @@ mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
@@ -287,7 +286,6 @@ mod tests {
                 align: 4096,
                 io_size: 4096 * KB,
             },
-            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 72f9aa8b..dae305e5 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -130,7 +130,6 @@ async fn test_store() {
             align: 4 * KB,
             io_size: 4 * KB,
         },
-        ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
@@ -159,7 +158,6 @@ async fn test_store_zstd() {
             align: 4 * KB,
             io_size: 4 * KB,
         },
-        ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
@@ -188,7 +186,6 @@ async fn test_store_lz4() {
             align: 4 * KB,
             io_size: 4 * KB,
         },
-        ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
@@ -217,7 +214,6 @@ async fn test_lazy_store() {
             align: 4 * KB,
             io_size: 4 * KB,
         },
-        ring_buffer_capacity: 16 * MB,
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
@@ -247,7 +243,6 @@ async fn test_runtime_store() {
                 align: 4 * KB,
                 io_size: 4 * KB,
             },
-            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
@@ -283,7 +278,6 @@ async fn test_runtime_lazy_store() {
                 align: 4 * KB,
                 io_size: 4 * KB,
             },
-            ring_buffer_capacity: 16 * MB,
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],

From 83c581ab8105d2d80c17a0b0db51f66b738caa05 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 15 Dec 2023 13:22:40 +0800
Subject: [PATCH 184/261] refactor: simplify cursor interface (#232)

---
 foyer-common/src/code.rs | 48 ----------------------------------------
 1 file changed, 48 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 86979876..4a363d23 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -91,15 +91,7 @@ impl<T: BufMut> BufMutExt for T {}
 pub trait Cursor: Send + Sync + 'static + std::io::Read + std::fmt::Debug {
     type T: Send + Sync + 'static;
 
-    fn inner(&self) -> &Self::T;
-
     fn into_inner(self) -> Self::T;
-
-    fn len(&self) -> usize;
-
-    fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
 }
 
 /// [`Key`] is required to implement [`Clone`].
@@ -213,17 +205,9 @@ macro_rules! def_cursor {
                 impl Cursor for [<PrimitiveCursor $id>] {
                     type T = $type;
 
-                    fn inner(&self) -> &Self::T {
-                        &self.inner
-                    }
-
                     fn into_inner(self) -> Self::T {
                         self.inner
                     }
-
-                    fn len(&self) -> usize {
-                        std::mem::size_of::<$type>()
-                    }
                 }
             )*
         }
@@ -325,17 +309,9 @@ impl Value for Vec<u8> {
 impl Cursor for std::io::Cursor<Vec<u8>> {
     type T = Vec<u8>;
 
-    fn inner(&self) -> &Self::T {
-        self.get_ref()
-    }
-
     fn into_inner(self) -> Self::T {
         self.into_inner()
     }
-
-    fn len(&self) -> usize {
-        self.get_ref().len()
-    }
 }
 
 impl Key for std::sync::Arc<Vec<u8>> {
@@ -403,17 +379,9 @@ impl std::io::Read for ArcVecU8Cursor {
 impl Cursor for ArcVecU8Cursor {
     type T = std::sync::Arc<Vec<u8>>;
 
-    fn inner(&self) -> &Self::T {
-        &self.inner
-    }
-
     fn into_inner(self) -> Self::T {
         self.inner
     }
-
-    fn len(&self) -> usize {
-        self.inner.len()
-    }
 }
 
 #[derive(Debug)]
@@ -428,15 +396,7 @@ impl std::io::Read for PrimitiveCursorVoid {
 impl Cursor for PrimitiveCursorVoid {
     type T = ();
 
-    fn inner(&self) -> &Self::T {
-        &()
-    }
-
     fn into_inner(self) -> Self::T {}
-
-    fn len(&self) -> usize {
-        0
-    }
 }
 
 impl Key for () {
@@ -491,15 +451,7 @@ impl<T: Send + Sync + 'static + std::fmt::Debug> std::io::Read for Unimplemented
 impl<T: Send + Sync + 'static + std::fmt::Debug> Cursor for UnimplementedCursor<T> {
     type T = T;
 
-    fn inner(&self) -> &Self::T {
-        unimplemented!()
-    }
-
     fn into_inner(self) -> Self::T {
         unimplemented!()
     }
-
-    fn len(&self) -> usize {
-        unimplemented!()
-    }
 }

From e69bb9a7411bf56e715b8ac300680c4bfbc3aecd Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 15 Dec 2023 15:50:30 +0800
Subject: [PATCH 185/261] chore: remove unused ring buffer (#234)

---
 foyer-storage/src/lib.rs  |   1 -
 foyer-storage/src/ring.rs | 450 --------------------------------------
 2 files changed, 451 deletions(-)
 delete mode 100644 foyer-storage/src/ring.rs

diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index b2bcac1e..94155679 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -39,7 +39,6 @@ pub mod reclaimer;
 pub mod region;
 pub mod region_manager;
 pub mod reinsertion;
-pub mod ring;
 pub mod runtime;
 pub mod storage;
 pub mod store;
diff --git a/foyer-storage/src/ring.rs b/foyer-storage/src/ring.rs
deleted file mode 100644
index 495bb56a..00000000
--- a/foyer-storage/src/ring.rs
+++ /dev/null
@@ -1,450 +0,0 @@
-//  Copyright 2023 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{
-    alloc::Global,
-    fmt::Debug,
-    ops::{Deref, DerefMut, Range},
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc,
-    },
-    time::{Duration, Instant},
-};
-
-use foyer_common::{bits::align_up, continuum::ContinuumUsize};
-use itertools::Itertools;
-
-use crate::{
-    catalog::Sequence,
-    device::BufferAllocator,
-    metrics::{Metrics, METRICS},
-};
-
-pub struct RingBuffer<A = Global>
-where
-    A: BufferAllocator,
-{
-    data: Vec<u8, A>,
-    align: usize,
-    capacity: usize,
-    blocks: usize,
-
-    allocated: AtomicUsize,
-
-    refs: Vec<Arc<AtomicUsize>>,
-
-    continuum: Arc<ContinuumUsize>,
-
-    metrics: Arc<Metrics>,
-}
-
-impl<A> Debug for RingBuffer<A>
-where
-    A: BufferAllocator,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("RingBuffer")
-            .field("align", &self.align)
-            .field("capacity", &self.capacity)
-            .field("blocks", &self.blocks)
-            .field("allocated", &self.allocated)
-            .finish()
-    }
-}
-
-impl RingBuffer<Global> {
-    /// `align` must be power of 2.
-    pub fn new(align: usize, capacity: usize) -> Self {
-        Self::new_in(align, capacity, Global)
-    }
-
-    /// `align` must be power of 2.
-    pub fn with_metrics(align: usize, capacity: usize, metrics: Arc<Metrics>) -> Self {
-        Self::with_metrics_in(align, capacity, metrics, Global)
-    }
-}
-
-impl<A> RingBuffer<A>
-where
-    A: BufferAllocator,
-{
-    /// `align` must be power of 2.
-    pub fn new_in(align: usize, capacity: usize, alloc: A) -> Self {
-        let metrics = Arc::new(METRICS.foyer(""));
-        Self::with_metrics_in(align, capacity, metrics, alloc)
-    }
-
-    /// `align` must be power of 2.
-    pub fn with_metrics_in(align: usize, capacity: usize, metrics: Arc<Metrics>, alloc: A) -> Self {
-        assert!(align.is_power_of_two());
-        let capacity = align_up(align, capacity);
-        let blocks = capacity / align;
-
-        let mut data = Vec::with_capacity_in(capacity, alloc);
-        unsafe { data.set_len(capacity) };
-
-        let allocated = AtomicUsize::new(0);
-
-        let continuum = Arc::new(ContinuumUsize::new(blocks));
-        let refs = (0..blocks)
-            .map(|_| Arc::new(AtomicUsize::default()))
-            .collect_vec();
-
-        Self {
-            data,
-            align,
-            capacity,
-            blocks,
-            allocated,
-            refs,
-            continuum,
-            metrics,
-        }
-    }
-
-    /// Allocate a mutable view with required size on the ring buffer.
-    ///
-    /// Returns `None` when the allocated buffer cross the boundary.
-    ///
-    /// When all views from an allocation are dropped, the buffer will be released.
-    #[tracing::instrument(skip(self))]
-    pub async fn allocate(self: &Arc<Self>, len: usize, sequence: Sequence) -> RingBufferViewMut {
-        loop {
-            if let Some(view) = self.allocate_inner(len, sequence).await {
-                return view;
-            }
-        }
-    }
-
-    #[tracing::instrument(skip(self))]
-    async fn allocate_inner(
-        self: &Arc<Self>,
-        len: usize,
-        sequence: Sequence,
-    ) -> Option<RingBufferViewMut> {
-        let len = align_up(self.align, len);
-        let offset = self.allocated.fetch_add(len, Ordering::SeqCst);
-
-        debug_assert_eq!(offset & (self.align - 1), 0);
-
-        let now = Instant::now();
-        loop {
-            self.continuum.advance();
-            if self.continuum.continuum() * self.align + self.capacity >= offset + len {
-                break;
-            }
-            tokio::time::sleep(Duration::from_micros(100)).await;
-        }
-        let elapsed = now.elapsed();
-        self.metrics
-            .inner_op_duration_wait_ring_buffer
-            .observe(elapsed.as_secs_f64());
-
-        if offset / self.capacity != (offset + len) / self.capacity {
-            debug_assert!(self.continuum.is_vacant(offset / self.align));
-
-            self.continuum
-                .submit_advance((offset / self.align)..((offset + len) / self.align));
-            return None;
-        }
-
-        let refs = self.refs(sequence);
-
-        let ring = Arc::clone(self);
-        let ring: Arc<dyn Ring> = ring;
-        Some(RingBufferViewMut::new(ring, offset, len, refs))
-    }
-
-    fn refs(&self, sequence: Sequence) -> &Arc<AtomicUsize> {
-        &self.refs[sequence as usize % self.blocks]
-    }
-
-    pub fn continuum(&self) -> usize {
-        self.continuum.continuum() * self.align
-    }
-
-    pub fn allocated(&self) -> usize {
-        self.allocated.load(Ordering::Relaxed)
-    }
-
-    pub fn advance(&self) {
-        self.continuum.advance();
-    }
-
-    /// Reutrns positive number means how many bytes remains,
-    /// returns negative number means how many bytes are waiting for allocation.
-    pub fn remains(&self) -> isize {
-        self.continuum() as isize - self.allocated() as isize
-    }
-}
-
-pub trait Ring: Send + Sync + 'static + Debug {
-    fn align(&self) -> usize;
-
-    fn ptr(&self, offset: usize) -> *mut u8;
-
-    fn release(&self, range: Range<usize>);
-}
-
-impl<A> Ring for RingBuffer<A>
-where
-    A: BufferAllocator,
-{
-    fn align(&self) -> usize {
-        self.align
-    }
-
-    fn ptr(&self, offset: usize) -> *mut u8 {
-        (self.data.as_ptr() as usize + (offset % self.capacity)) as *mut u8
-    }
-
-    fn release(&self, range: Range<usize>) {
-        let start = range.start / self.align;
-        let end = range.end / self.align;
-        debug_assert!(self.continuum.is_vacant(start));
-        self.continuum.submit_advance(start..end);
-    }
-}
-
-/// # Safety
-///
-/// The underlying buffer of [`ViewMut`] must be valid during its lifetime.
-#[derive(Debug)]
-pub struct RingBufferViewMut {
-    ring: Arc<dyn Ring>,
-    ptr: *mut u8,
-    offset: usize,
-    len: usize,
-    capacity: usize,
-    refs: Arc<AtomicUsize>,
-}
-
-impl RingBufferViewMut {
-    fn new(ring: Arc<dyn Ring>, offset: usize, len: usize, refs: &Arc<AtomicUsize>) -> Self {
-        refs.fetch_add(1, Ordering::AcqRel);
-        let ptr = ring.ptr(offset);
-        Self {
-            ring,
-            ptr,
-            offset,
-            len,
-            capacity: len,
-            refs: Arc::clone(refs),
-        }
-    }
-
-    /// Shrink the accessable area of [`ViewMut`].
-    ///
-    /// `shrink` will not release the underlying buffer size.
-    pub fn shrink_to(&mut self, len: usize) {
-        debug_assert!(len <= self.capacity);
-        self.len = len;
-    }
-
-    pub fn aligned(&self) -> usize {
-        align_up(self.ring.align(), self.len)
-    }
-
-    pub fn freeze(self) -> RingBufferView {
-        RingBufferView::from(self)
-    }
-}
-
-impl Deref for RingBufferViewMut {
-    type Target = [u8];
-
-    fn deref(&self) -> &Self::Target {
-        unsafe { core::slice::from_raw_parts(self.ptr, self.len) }
-    }
-}
-
-impl DerefMut for RingBufferViewMut {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) }
-    }
-}
-
-impl Drop for RingBufferViewMut {
-    fn drop(&mut self) {
-        if self.refs.fetch_sub(1, Ordering::AcqRel) == 1 {
-            self.ring.release(self.offset..self.offset + self.capacity);
-        }
-    }
-}
-
-unsafe impl Send for RingBufferViewMut {}
-unsafe impl Sync for RingBufferViewMut {}
-
-/// # Safety
-///
-/// The underlying buffer of [`View`] must be valid during its lifetime.
-#[derive(Debug)]
-pub struct RingBufferView {
-    view: RingBufferViewMut,
-}
-
-impl From<RingBufferViewMut> for RingBufferView {
-    fn from(view: RingBufferViewMut) -> Self {
-        Self { view }
-    }
-}
-
-impl Deref for RingBufferView {
-    type Target = [u8];
-
-    fn deref(&self) -> &Self::Target {
-        self.view.deref()
-    }
-}
-
-impl Clone for RingBufferView {
-    fn clone(&self) -> Self {
-        self.view.refs.fetch_add(1, Ordering::AcqRel);
-        let view = RingBufferViewMut {
-            ring: self.view.ring.clone(),
-            ptr: self.view.ptr,
-            offset: self.view.offset,
-            len: self.view.len,
-            capacity: self.view.capacity,
-            refs: self.view.refs.clone(),
-        };
-        Self { view }
-    }
-}
-
-impl RingBufferView {
-    pub fn aligned(&self) -> usize {
-        self.view.aligned()
-    }
-}
-
-unsafe impl Send for RingBufferView {}
-unsafe impl Sync for RingBufferView {}
-
-#[cfg(test)]
-mod tests {
-
-    use std::{
-        collections::BTreeMap,
-        future::{poll_fn, Future},
-        ops::Range,
-        pin::pin,
-        sync::atomic::AtomicU64,
-        task::{Poll, Poll::Pending},
-        thread::available_parallelism,
-        time::Instant,
-    };
-
-    use bytes::BufMut;
-    use rand::{rngs::OsRng, Rng};
-
-    use super::*;
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_ring() {
-        const ALIGN: usize = 4096; // 4 KiB
-        const CAPACITY: usize = 16 * 1024 * 1024; // 16 MiB
-
-        let ring = Arc::new(RingBuffer::new(ALIGN, CAPACITY));
-        let sequence = Arc::new(AtomicU64::default());
-
-        let mut views = BTreeMap::new();
-        for i in 0..15 {
-            let seq = sequence.fetch_add(1, Ordering::Relaxed);
-            let view = ring
-                .allocate_inner(1024 * 1024 - 3000 /* ~ 1 MiB */, seq)
-                .await
-                .unwrap();
-            assert_eq!(view.offset, i * 1024 * 1024);
-            assert_eq!(view.capacity, 1024 * 1024);
-            views.insert(seq, view);
-        }
-        let seq = sequence.fetch_add(1, Ordering::Relaxed);
-
-        let mut future = pin!(ring.allocate_inner(2 * 1024 * 1024 - 3000 /* ~ 2 MiB */, seq));
-        assert!(matches! { poll_fn(|cx| Poll::Ready(future.as_mut().poll(cx))).await, Pending });
-        views.remove(&0).unwrap();
-        let res = future.await;
-        assert!(res.is_none());
-
-        let mut future = pin!(ring.allocate_inner(2 * 1024 * 1024 - 3000 /* ~ 2 MiB */, seq));
-        assert!(matches! { poll_fn(|cx| Poll::Ready(future.as_mut().poll(cx))).await, Pending });
-        views.remove(&2).unwrap();
-        views.remove(&1).unwrap();
-        let view = future.await.unwrap();
-        assert_eq!(view.offset, 17 * 1024 * 1024);
-        assert_eq!(view.capacity, 2 * 1024 * 1024);
-        views.insert(seq, view);
-
-        drop(views);
-    }
-
-    async fn test_ring_concurrent_case(capacity: usize, concurrency: usize, loops: usize) {
-        const ALIGN: usize = 4096; // 4 KiB
-        const SIZE: Range<usize> = 16 * 1024..256 * 1024; // 16 KiB ~ 128 KiB
-
-        let ring = Arc::new(RingBuffer::new(ALIGN, capacity));
-        let sequence = Arc::new(AtomicU64::default());
-
-        let tasks = (0..concurrency)
-            .map(|_| {
-                let ring = ring.clone();
-                let sequence = sequence.clone();
-                async move {
-                    for i in 0..loops {
-                        let seq = sequence.fetch_add(1, Ordering::Relaxed);
-                        let size = OsRng.gen_range(SIZE);
-                        let mut view = ring.allocate(size, seq).await;
-                        tokio::time::sleep(Duration::from_millis(OsRng.gen_range(0..10))).await;
-                        let data = vec![i as u8; size];
-                        view.as_mut().put_slice(&data);
-                        let view = view.freeze();
-                        assert!(view[..size] == data);
-                        drop(view);
-                    }
-                }
-            })
-            .collect_vec();
-        let handles = tasks.into_iter().map(tokio::spawn).collect_vec();
-        for handle in handles {
-            handle.await.unwrap();
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_ring_concurrent_small() {
-        test_ring_concurrent_case(16 * 1024 * 1024, 16, 100).await;
-    }
-
-    #[ignore]
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_ring_concurrent_large() {
-        let concurrency = available_parallelism().unwrap().get() * 64;
-
-        let now = Instant::now();
-        test_ring_concurrent_case(
-            128 * 1024 * 1024,
-            available_parallelism().unwrap().get() * 64,
-            1000,
-        )
-        .await;
-        let elapsed = now.elapsed();
-
-        println!("========== ring current fuzzy begin ==========");
-        println!("concurrency: {concurrency}");
-        println!("elapsed: {elapsed:?}");
-        println!("=========== ring current fuzzy end ===========");
-    }
-}

From 1103266a1d1c9876c73c877920cc7618c21c5a41 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 18 Dec 2023 12:50:32 +0800
Subject: [PATCH 186/261] chore: bump version to 0.3 (#235)

---
 foyer-common/Cargo.toml        | 2 +-
 foyer-intrusive/Cargo.toml     | 2 +-
 foyer-storage-bench/Cargo.toml | 6 +++---
 foyer-storage/Cargo.toml       | 4 ++--
 foyer/Cargo.toml               | 6 +++---
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 055acb85..22d2e5c3 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-common"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "common utils for foyer - the hybrid cache for Rust"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 26980ace..35558b8e 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -15,7 +15,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { version = "0.1", path = "../foyer-common" }
+foyer-common = { version = "0.2", path = "../foyer-common" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 memoffset = "0.9"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 119b5b11..b52333ff 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage-bench"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine bench tool for foyer - the hybrid cache for Rust"
@@ -17,9 +17,9 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
-foyer-common = { version = "0.1", path = "../foyer-common" }
+foyer-common = { version = "0.2", path = "../foyer-common" }
 foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.1", path = "../foyer-storage" }
+foyer-storage = { version = "0.2", path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index dc6c68e1..a49d9952 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
@@ -18,7 +18,7 @@ bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { version = "0.1", path = "../foyer-common" }
+foyer-common = { version = "0.2", path = "../foyer-common" }
 foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 51335401..66af894f 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -13,7 +13,7 @@ homepage = "https://github.com/mrcroxx/foyer"
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
-foyer-common = { version = "0.1", path = "../foyer-common" }
+foyer-common = { version = "0.2", path = "../foyer-common" }
 foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.1", path = "../foyer-storage" }
+foyer-storage = { version = "0.2", path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }

From 569f72ccad1da7226c34fda6b9bd71871d53839b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 18 Dec 2023 14:42:06 +0800
Subject: [PATCH 187/261] chore: update metrics, add entry size histogram
 (#237)

---
 etc/grafana/dashboards/foyer.json |  2 +-
 foyer-storage/src/generic.rs      |  7 +++---
 foyer-storage/src/metrics.rs      | 37 ++++++++++++++++++++-----------
 3 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/etc/grafana/dashboards/foyer.json b/etc/grafana/dashboards/foyer.json
index 2923d0b4..bf43af6e 100644
--- a/etc/grafana/dashboards/foyer.json
+++ b/etc/grafana/dashboards/foyer.json
@@ -1 +1 @@
-{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Write Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":10,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Read Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":9,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":25},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":33},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":33},"id":11,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_inner_bytes) by (foyer, component, extra) ","instant":false,"legendFormat":"{{foyer}} {{component}} {{extra}}","range":true,"refId":"A"}],"title":"Inner Size","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""}
+{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_slow_op_duration_count[$__rate_interval])) by (foyer, op, extra)","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Slow Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":3,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"insert|remove\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Write Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":10,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{op=~\"lookup\"}[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Op Read Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"hit\"}[$__rate_interval])) by (foyer) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\", extra=\"miss\"}[$__rate_interval])) by (foyer)) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":25},"id":9,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket [$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":25},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Slow Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":33},"id":12,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_entry_bytes_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","instant":false,"legendFormat":"p50 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_entry_bytes_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p90 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_entry_bytes_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"p99 - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_entry_bytes_bucket[$__rate_interval])) by (le, foyer, op, extra)) ","hide":false,"instant":false,"legendFormat":"pmax - {{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"D"}],"title":"Entry Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":33},"id":11,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_inner_bytes) by (foyer, component, extra) ","instant":false,"legendFormat":"{{foyer}} {{component}} {{extra}}","range":true,"refId":"A"}],"title":"Inner Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":41},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_total_bytes) by (foyer) ","instant":false,"legendFormat":"{{foyer}} foyer storage","range":true,"refId":"A"}],"title":"Size","type":"timeseries"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""}
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 9f34f64c..e6e6274c 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -402,7 +402,6 @@ where
 
                 Ok(Some(value))
             }
-            // read from region
             crate::catalog::Index::Region { view } => {
                 let region = view.id();
 
@@ -580,10 +579,12 @@ where
         }
 
         // record aligned header + key + value size for metrics
-        self.inner.metrics.op_bytes_insert.inc_by(bits::align_up(
+        let len = bits::align_up(
             self.inner.device.align(),
             EntryHeader::serialized_len() + key.serialized_len() + value.serialized_len(),
-        ) as u64);
+        );
+        self.inner.metrics.op_bytes_insert.inc_by(len as u64);
+        self.inner.metrics.insert_entry_bytes.observe(len as f64);
 
         self.inner.catalog.insert(
             key.clone(),
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index f6ea99fe..9a67a6f8 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -16,9 +16,9 @@ use std::sync::{LazyLock, OnceLock};
 
 use prometheus::{
     core::{AtomicU64, GenericGauge, GenericGaugeVec},
-    opts, register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
-    register_int_gauge_vec_with_registry, Histogram, HistogramVec, IntCounter, IntCounterVec,
-    IntGauge, IntGaugeVec, Registry,
+    exponential_buckets, opts, register_histogram_vec_with_registry,
+    register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry, Histogram,
+    HistogramVec, IntCounter, IntCounterVec, IntGaugeVec, Registry,
 };
 type UintGaugeVec = GenericGaugeVec<AtomicU64>;
 type UintGauge = GenericGauge<AtomicU64>;
@@ -67,8 +67,10 @@ pub struct GlobalMetrics {
     op_bytes: IntCounterVec,
     total_bytes: UintGaugeVec,
 
+    entry_bytes: HistogramVec,
+
     inner_op_duration: HistogramVec,
-    inner_bytes: IntGaugeVec,
+    _inner_bytes: IntGaugeVec,
 }
 
 impl Default for GlobalMetrics {
@@ -113,6 +115,15 @@ impl GlobalMetrics {
         )
         .unwrap();
 
+        let entry_bytes = register_histogram_vec_with_registry!(
+            "foyer_storage_entry_bytes",
+            "foyer storage entry bytes",
+            &["foyer", "op", "extra"],
+            exponential_buckets(1.0, 2.0, 32).unwrap(),
+            registry,
+        )
+        .unwrap();
+
         let inner_op_duration = register_histogram_vec_with_registry!(
             "foyer_storage_inner_op_duration",
             "foyer storage inner op duration",
@@ -136,8 +147,10 @@ impl GlobalMetrics {
             op_bytes,
             total_bytes,
 
+            entry_bytes,
+
             inner_op_duration,
-            inner_bytes,
+            _inner_bytes: inner_bytes,
         }
     }
 
@@ -164,14 +177,14 @@ pub struct Metrics {
 
     pub total_bytes: UintGauge,
 
+    pub insert_entry_bytes: Histogram,
+
     pub inner_op_duration_acquire_clean_region: Histogram,
     pub inner_op_duration_acquire_clean_buffer: Histogram,
     pub inner_op_duration_wait_ring_buffer: Histogram,
     pub inner_op_duration_update_catalog: Histogram,
     pub inner_op_duration_entry_flush: Histogram,
     pub inner_op_duration_flusher_handle: Histogram,
-
-    pub inner_bytes_ring_buffer_remains: IntGauge,
 }
 
 impl Metrics {
@@ -204,6 +217,8 @@ impl Metrics {
 
         let total_bytes = global.total_bytes.with_label_values(&[foyer]);
 
+        let insert_entry_bytes = global.entry_bytes.with_label_values(&[foyer, "insert", ""]);
+
         let inner_op_duration_acquire_clean_region =
             global
                 .inner_op_duration
@@ -229,10 +244,6 @@ impl Metrics {
                 .inner_op_duration
                 .with_label_values(&[foyer, "flusher_handle", ""]);
 
-        let inner_bytes_ring_buffer_remains = global
-            .inner_bytes
-            .with_label_values(&[foyer, "ring", "remains"]);
-
         Self {
             op_duration_insert_inserted,
             op_duration_insert_filtered,
@@ -250,14 +261,14 @@ impl Metrics {
 
             total_bytes,
 
+            insert_entry_bytes,
+
             inner_op_duration_acquire_clean_region,
             inner_op_duration_acquire_clean_buffer,
             inner_op_duration_wait_ring_buffer,
             inner_op_duration_update_catalog,
             inner_op_duration_entry_flush,
             inner_op_duration_flusher_handle,
-
-            inner_bytes_ring_buffer_remains,
         }
     }
 }

From e12ff6838c813dd3b99fd0747708210eefccc707 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 18 Dec 2023 14:50:13 +0800
Subject: [PATCH 188/261] chore: bump foyer-storage to 0.2.1 (#238)

---
 foyer-storage/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index a49d9952..2de23e52 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.2.0"
+version = "0.2.1"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"

From 01b8cc806d29ea67898fb40505b520bcb39cf78a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 19 Dec 2023 17:15:33 +0800
Subject: [PATCH 189/261] fix: restore writer drop metrics (#222)

---
 foyer-storage/src/generic.rs    | 94 ++++++++++++++++++---------------
 foyer-workspace-hack/Cargo.toml |  1 +
 2 files changed, 51 insertions(+), 44 deletions(-)

diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index e6e6274c..745a1914 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -544,7 +544,7 @@ where
 
     fn judge_inner(&self, writer: &mut GenericStoreWriter<K, V, D, EP, EL>) {
         for (index, admission) in self.inner.admissions.iter().enumerate() {
-            let judge = admission.judge(&writer.key, writer.weight);
+            let judge = admission.judge(writer.key.as_ref().unwrap(), writer.weight);
             writer.judges.set(index, judge);
         }
         writer.is_judged = true;
@@ -571,7 +571,7 @@ where
         };
 
         writer.is_inserted = true;
-        let key = writer.key;
+        let key = writer.key.take().unwrap();
 
         for (i, admission) in self.inner.admissions.iter().enumerate() {
             let judge = writer.judges.get(i);
@@ -626,7 +626,8 @@ where
     EL: Link,
 {
     store: GenericStore<K, V, D, EP, EL>,
-    key: K,
+    /// `key` is always `Some` before `apply_writer`.
+    key: Option<K>,
     weight: usize,
 
     sequence: Option<Sequence>,
@@ -655,7 +656,7 @@ where
         let compression = store.inner.compression;
         Self {
             store,
-            key,
+            key: Some(key),
             weight,
             sequence: None,
             judges,
@@ -728,45 +729,50 @@ where
     }
 }
 
-// impl<K, V, D, EP, EL> Drop for GenericStoreWriter<K, V, D, EP, EL>
-// where
-//     K: Key,
-//     V: Value,
-//     D: Device,
-//     EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
-//     EL: Link,
-// {
-//     fn drop(&mut self) {
-//         if !self.is_inserted {
-//             self.store
-//                 .inner
-//                 .metrics
-//                 .op_duration_insert_dropped
-//                 .observe(self.duration.as_secs_f64());
-//             let mut filtered = false;
-//             if self.is_judged {
-//                 for (i, admission) in self.store.inner.admissions.iter().enumerate() {
-//                     let judge = self.judges.get(i);
-//                     admission.on_drop(&self.key, self.weight, &self.store.inner.metrics, judge);
-//                 }
-//                 filtered = !self.judge();
-//             }
-//             if filtered {
-//                 self.store
-//                     .inner
-//                     .metrics
-//                     .op_duration_insert_filtered
-//                     .observe(self.duration.as_secs_f64());
-//             } else {
-//                 self.store
-//                     .inner
-//                     .metrics
-//                     .op_duration_insert_dropped
-//                     .observe(self.duration.as_secs_f64());
-//             }
-//         }
-//     }
-// }
+impl<K, V, D, EP, EL> Drop for GenericStoreWriter<K, V, D, EP, EL>
+where
+    K: Key,
+    V: Value,
+    D: Device,
+    EP: EvictionPolicy<Adapter = RegionEpItemAdapter<EL>>,
+    EL: Link,
+{
+    fn drop(&mut self) {
+        if !self.is_inserted {
+            debug_assert!(self.key.is_some());
+
+            self.store
+                .inner
+                .metrics
+                .op_duration_insert_dropped
+                .observe(self.duration.as_secs_f64());
+
+            // make sure each key after `judge` will call either `on_insert` or `on_drop`.
+            let mut filtered = false;
+            if self.is_judged {
+                filtered = !self.judge();
+                for (i, admission) in self.store.inner.admissions.iter().enumerate() {
+                    let judge = self.judges.get(i);
+                    admission.on_drop(self.key.as_ref().unwrap(), self.weight, judge);
+                }
+            }
+
+            if filtered {
+                self.store
+                    .inner
+                    .metrics
+                    .op_duration_insert_filtered
+                    .observe(self.duration.as_secs_f64());
+            } else {
+                self.store
+                    .inner
+                    .metrics
+                    .op_duration_insert_dropped
+                    .observe(self.duration.as_secs_f64());
+            }
+        }
+    }
+}
 
 const ENTRY_MAGIC: u32 = 0x97_03_27_00;
 const ENTRY_MAGIC_MASK: u32 = 0xFF_FF_FF_00;
@@ -1028,7 +1034,7 @@ where
     type Value = V;
 
     fn key(&self) -> &Self::Key {
-        &self.key
+        self.key.as_ref().unwrap()
     }
 
     fn weight(&self) -> usize {
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 6d6f2491..41e1ac8f 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -30,6 +30,7 @@ memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
+socket2 = { version = "0.5", default-features = false, features = ["all"] }
 tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 tokio-util = { version = "0.7", features = ["codec", "io"] }
 tower = { version = "0.4", features = ["balance", "buffer", "limit", "timeout", "util"] }

From 360ee5fff9f259ecf10442919936ff18ab79f341 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 19 Dec 2023 18:32:01 +0800
Subject: [PATCH 190/261] feat: add insert if not exist with callback
 interface, add uts (#239)

---
 foyer-storage/src/storage.rs | 177 +++++++++++++++++++++++++++++++++++
 1 file changed, 177 insertions(+)

diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index d02cb553..282c3e43 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -219,6 +219,23 @@ pub trait AsyncStorageExt: Storage {
             }
         });
     }
+
+    fn insert_if_not_exists_async_with_callback<F, FU>(
+        &self,
+        key: Self::Key,
+        value: Self::Value,
+        f: F,
+    ) where
+        F: FnOnce(Result<bool>) -> FU + Send + 'static,
+        FU: Future<Output = ()> + Send + 'static,
+    {
+        let store = self.clone();
+        tokio::spawn(async move {
+            let res = store.insert_if_not_exists(key, value).await;
+            let future = f(res);
+            future.await;
+        });
+    }
 }
 
 impl<S: Storage> AsyncStorageExt for S {}
@@ -307,3 +324,163 @@ pub trait ForceStorageExt: Storage {
 }
 
 impl<S> ForceStorageExt for S where S: Storage {}
+
+#[cfg(test)]
+mod tests {
+    //! storage interface test
+
+    use std::{path::Path, time::Duration};
+
+    use foyer_intrusive::eviction::fifo::FifoConfig;
+
+    use super::*;
+    use crate::{
+        device::fs::FsDeviceConfig,
+        store::{FifoFsStore, FifoFsStoreConfig},
+    };
+
+    const KB: usize = 1024;
+    const MB: usize = 1024 * 1024;
+
+    fn config_for_test(dir: impl AsRef<Path>) -> FifoFsStoreConfig<u64, Vec<u8>> {
+        FifoFsStoreConfig {
+            name: "".to_string(),
+            eviction_config: FifoConfig,
+            device_config: FsDeviceConfig {
+                dir: dir.as_ref().into(),
+                capacity: 4 * MB,
+                file_capacity: MB,
+                align: 4 * KB,
+                io_size: 4 * KB,
+            },
+            catalog_bits: 1,
+            admissions: vec![],
+            reinsertions: vec![],
+            flusher_buffer_size: 0,
+            flushers: 1,
+            reclaimers: 1,
+            clean_region_threshold: 1,
+            recover_concurrency: 2,
+            compression: Compression::None,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_storage() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let config = config_for_test(tempdir.path());
+
+        let storage = FifoFsStore::open(config).await.unwrap();
+        assert!(storage.is_ready());
+
+        assert!(!storage.exists(&1).unwrap());
+
+        let mut writer = storage.writer(1, KB);
+        assert_eq!(writer.key(), &1);
+        assert_eq!(writer.weight(), KB);
+        assert!(writer.judge());
+        assert_eq!(writer.compression(), Compression::None);
+        writer.set_compression(Compression::Lz4);
+        assert_eq!(writer.compression(), Compression::Lz4);
+        writer.force();
+        assert!(writer.finish(vec![b'x'; KB]).await.unwrap());
+
+        assert!(storage.exists(&1).unwrap());
+        assert_eq!(storage.lookup(&1).await.unwrap().unwrap(), vec![b'x'; KB]);
+
+        assert!(storage.remove(&1).unwrap());
+        assert!(!storage.exists(&1).unwrap());
+        assert!(!storage.remove(&1).unwrap());
+
+        storage.clear().unwrap();
+        storage.close().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_storage_ext() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let config = config_for_test(tempdir.path());
+
+        let storage = FifoFsStore::open(config).await.unwrap();
+
+        assert!(storage.insert(1, vec![b'x'; KB]).await.unwrap());
+        assert!(storage.exists(&1).unwrap());
+
+        assert!(!storage
+            .insert_if_not_exists(1, vec![b'x'; KB])
+            .await
+            .unwrap());
+        assert!(storage
+            .insert_if_not_exists(2, vec![b'x'; KB])
+            .await
+            .unwrap());
+        assert!(storage.exists(&2).unwrap());
+
+        assert!(storage
+            .insert_with(3, || { Ok(vec![b'x'; KB]) }, KB)
+            .await
+            .unwrap());
+        assert!(storage.exists(&3).unwrap());
+
+        assert!(storage
+            .insert_with_future(4, || { async move { Ok(vec![b'x'; KB]) } }, KB)
+            .await
+            .unwrap());
+        assert!(storage.exists(&4).unwrap());
+
+        assert!(!storage
+            .insert_if_not_exists_with(4, || { Ok(vec![b'x'; KB]) }, KB)
+            .await
+            .unwrap());
+        assert!(storage
+            .insert_if_not_exists_with(5, || { Ok(vec![b'x'; KB]) }, KB)
+            .await
+            .unwrap());
+        assert!(storage.exists(&5).unwrap());
+
+        assert!(!storage
+            .insert_if_not_exists_with_future(5, || { async move { Ok(vec![b'x'; KB]) } }, KB)
+            .await
+            .unwrap());
+        assert!(storage
+            .insert_if_not_exists_with_future(6, || { async move { Ok(vec![b'x'; KB]) } }, KB)
+            .await
+            .unwrap());
+        assert!(storage.exists(&6).unwrap());
+    }
+
+    async fn exists_with_retry(
+        storage: &impl Storage<Key = u64, Value = Vec<u8>>,
+        key: &u64,
+    ) -> bool {
+        tokio::time::sleep(Duration::from_millis(1)).await;
+        for _ in 0..10 {
+            if storage.exists(key).unwrap() {
+                return true;
+            };
+            tokio::time::sleep(Duration::from_millis(10)).await
+        }
+        false
+    }
+
+    #[tokio::test]
+    async fn test_async_storage_ext() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let config = config_for_test(tempdir.path());
+
+        let storage = FifoFsStore::open(config).await.unwrap();
+
+        storage.insert_async(1, vec![b'x'; KB]);
+        assert!(exists_with_retry(&storage, &1).await);
+
+        storage.insert_if_not_exists_async(2, vec![b'x'; KB]);
+        assert!(exists_with_retry(&storage, &2).await);
+
+        storage.insert_if_not_exists_async_with_callback(2, vec![b'x'; KB], |res| async move {
+            assert!(!res.unwrap());
+        });
+        storage.insert_if_not_exists_async_with_callback(3, vec![b'x'; KB], |res| async move {
+            assert!(res.unwrap());
+        });
+    }
+}

From 5f2a760306e514c55e0754d3148e6f2f0ffb833a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 19 Dec 2023 22:35:41 +0800
Subject: [PATCH 191/261] chore: add CHANGELOG.md (#240)

---
 CHANGELOG.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 CHANGELOG.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 00000000..344e76b2
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,72 @@
+## 2023-12-18
+
+| crate | version |
+| - | - |
+| foyer-storage | 0.2.1 |
+
+<details>
+
+- Introduce the entry size histogram, update metrics.
+
+</details>
+
+## 2023-12-18
+
+| crate | version |
+| - | - |
+| foyer | 0.3.0 |
+| foyer-common | 0.2.0 |
+| foyer-storage | 0.2.0 |
+| foyer-storage-bench | 0.2.0 |
+
+<details>
+
+- Introduce the associated type `Cursor` for trait `Key` and `Value` to reduce unnecessary buffer copy if possible.
+- Remove the ring buffer and continuum tracker for they are no longer needed.
+- Update the configuration of the storage engine and the benchmark tool.
+
+</details>
+
+## 2023-11-29
+
+| crate | version |
+| - | - |
+| foyer | 0.2.0 |
+| foyer-common | 0.1.0 |
+| foyer-intrusive | 0.1.0 |
+| foyer-storage | 0.1.0 |
+| foyer-storage-bench | 0.1.0 |
+| foyer-workspace-hack | 0.1.0 |
+
+<details>
+
+The first version that can be used as file cache.
+
+The write model and the design of storage engine has been switched from CacheLib navy version to the ring buffer version (which was highly inspired by MySQL 8.0 link_buf).
+
+Introduces `Store`, `RuntimeStore`, `LazyStore` to simplify usage. In most cases, `RuntimeStore` is preferred to use a dedicated tokio runtime to serve **foyer** to avoid the influence to the user's runtime. If lazy-load is needed, use `RuntimeLazyStore` instead.
+
+The implementation of **foyer** is separated into multiple crates. But importing `foyer` is enought for it re-exports the crates that **foyer**'s user needs.
+
+Brief description about the subcrates:
+
+- foyer-common: Provide basic data structures and algorithms.
+- foyer-intrusive: Provide intrusive containers for implementing eviction lists and collections. Intrisive data structures provide the ability to implement low-cost multi-index data structures, which will be used for the memory cache in future.
+- foyer-storage: Provide the file cache storage engine and wrappers to simplify the code.
+- foyer-storage-bench: Runnable benchmark tool for the file cache storage engine.
+- foyer-workspace-hack: Generated by [hakari](https://crates.io/crates/hakari) to prevent building each crate from triggering building from scratch.
+
+</details>
+
+
+## 2023-05-12
+
+| crate | version |
+| - | - |
+| foyer | 0.1.0 |
+
+<details>
+
+Initial version with just bacis interfaces.
+
+</details>

From c6836a35174fcfb3ec23ef0d119de81457f34737 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 20 Dec 2023 09:23:15 +0800
Subject: [PATCH 192/261] feat: add insert_async_with_callback (#241)

---
 foyer-storage/src/storage.rs | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 282c3e43..9b6cb0c1 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -220,6 +220,19 @@ pub trait AsyncStorageExt: Storage {
         });
     }
 
+    fn insert_async_with_callback<F, FU>(&self, key: Self::Key, value: Self::Value, f: F)
+    where
+        F: FnOnce(Result<bool>) -> FU + Send + 'static,
+        FU: Future<Output = ()> + Send + 'static,
+    {
+        let store = self.clone();
+        tokio::spawn(async move {
+            let res = store.insert(key, value).await;
+            let future = f(res);
+            future.await;
+        });
+    }
+
     fn insert_if_not_exists_async_with_callback<F, FU>(
         &self,
         key: Self::Key,
@@ -476,10 +489,14 @@ mod tests {
         storage.insert_if_not_exists_async(2, vec![b'x'; KB]);
         assert!(exists_with_retry(&storage, &2).await);
 
-        storage.insert_if_not_exists_async_with_callback(2, vec![b'x'; KB], |res| async move {
-            assert!(!res.unwrap());
+        storage.insert_async_with_callback(3, vec![b'x'; KB], |res| async move {
+            assert!(res.unwrap());
         });
+
         storage.insert_if_not_exists_async_with_callback(3, vec![b'x'; KB], |res| async move {
+            assert!(!res.unwrap());
+        });
+        storage.insert_if_not_exists_async_with_callback(4, vec![b'x'; KB], |res| async move {
             assert!(res.unwrap());
         });
     }

From 130c7ab502f7d778b21d48b3130895d8637adb50 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 20 Dec 2023 09:34:27 +0800
Subject: [PATCH 193/261] Revert "feat: add insert_async_with_callback" (#242)

---
 foyer-storage/src/storage.rs | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 9b6cb0c1..282c3e43 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -220,19 +220,6 @@ pub trait AsyncStorageExt: Storage {
         });
     }
 
-    fn insert_async_with_callback<F, FU>(&self, key: Self::Key, value: Self::Value, f: F)
-    where
-        F: FnOnce(Result<bool>) -> FU + Send + 'static,
-        FU: Future<Output = ()> + Send + 'static,
-    {
-        let store = self.clone();
-        tokio::spawn(async move {
-            let res = store.insert(key, value).await;
-            let future = f(res);
-            future.await;
-        });
-    }
-
     fn insert_if_not_exists_async_with_callback<F, FU>(
         &self,
         key: Self::Key,
@@ -489,14 +476,10 @@ mod tests {
         storage.insert_if_not_exists_async(2, vec![b'x'; KB]);
         assert!(exists_with_retry(&storage, &2).await);
 
-        storage.insert_async_with_callback(3, vec![b'x'; KB], |res| async move {
-            assert!(res.unwrap());
-        });
-
-        storage.insert_if_not_exists_async_with_callback(3, vec![b'x'; KB], |res| async move {
+        storage.insert_if_not_exists_async_with_callback(2, vec![b'x'; KB], |res| async move {
             assert!(!res.unwrap());
         });
-        storage.insert_if_not_exists_async_with_callback(4, vec![b'x'; KB], |res| async move {
+        storage.insert_if_not_exists_async_with_callback(3, vec![b'x'; KB], |res| async move {
             assert!(res.unwrap());
         });
     }

From e39221fd0d8ffdf741a2b29138b0536c951473a4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 20 Dec 2023 09:37:58 +0800
Subject: [PATCH 194/261] feat: add insert_async_with_callback (#243)

---
 foyer-storage/src/storage.rs | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 282c3e43..9ae396de 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -220,6 +220,19 @@ pub trait AsyncStorageExt: Storage {
         });
     }
 
+    fn insert_async_with_callback<F, FU>(&self, key: Self::Key, value: Self::Value, f: F)
+    where
+        F: FnOnce(Result<bool>) -> FU + Send + 'static,
+        FU: Future<Output = ()> + Send + 'static,
+    {
+        let store = self.clone();
+        tokio::spawn(async move {
+            let res = store.insert(key, value).await;
+            let future = f(res);
+            future.await;
+        });
+    }
+
     fn insert_if_not_exists_async_with_callback<F, FU>(
         &self,
         key: Self::Key,
@@ -329,9 +342,10 @@ impl<S> ForceStorageExt for S where S: Storage {}
 mod tests {
     //! storage interface test
 
-    use std::{path::Path, time::Duration};
+    use std::{path::Path, sync::Arc, time::Duration};
 
     use foyer_intrusive::eviction::fifo::FifoConfig;
+    use tokio::sync::Barrier;
 
     use super::*;
     use crate::{
@@ -476,10 +490,18 @@ mod tests {
         storage.insert_if_not_exists_async(2, vec![b'x'; KB]);
         assert!(exists_with_retry(&storage, &2).await);
 
-        storage.insert_if_not_exists_async_with_callback(2, vec![b'x'; KB], |res| async move {
-            assert!(!res.unwrap());
+        let barrier = Arc::new(Barrier::new(2));
+        let b = barrier.clone();
+        storage.insert_async_with_callback(3, vec![b'x'; KB], |res| async move {
+            assert!(res.unwrap());
+            b.wait().await;
         });
+        barrier.wait().await;
+
         storage.insert_if_not_exists_async_with_callback(3, vec![b'x'; KB], |res| async move {
+            assert!(!res.unwrap());
+        });
+        storage.insert_if_not_exists_async_with_callback(4, vec![b'x'; KB], |res| async move {
             assert!(res.unwrap());
         });
     }

From 9d2a6f90cad1b79c1d467d3a054e72e313e893ac Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 20 Dec 2023 09:47:52 +0800
Subject: [PATCH 195/261] chore: bump foyer-storage to 0.2.2 (#244)

---
 CHANGELOG.md             | 13 +++++++++++++
 foyer-storage/Cargo.toml |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 344e76b2..4b69ea84 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,16 @@
+## 2023-12-20
+
+| crate | version |
+| - | - |
+| foyer-storage | 0.2.2 |
+
+<details>
+
+- Fix metrics for writer dropping.
+- Add interface `insert_async_with_callback` and `insert_if_not_exists_async_with_callback` for callers to get the insert result.
+
+</details>
+
 ## 2023-12-18
 
 | crate | version |
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 2de23e52..6764398c 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.2.1"
+version = "0.2.2"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"

From fc7471d726adb7600e3bf52fdb36c86a1f48d101 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 21 Dec 2023 18:16:00 +0800
Subject: [PATCH 196/261] chore: add typos check and fix typos (#248)

---
 .github/template/template.yml       | 2 ++
 .github/workflows/main.yml          | 2 ++
 .github/workflows/pull-request.yml  | 2 ++
 CHANGELOG.md                        | 2 +-
 Makefile                            | 1 +
 foyer-intrusive/src/core/adapter.rs | 6 +++---
 foyer-intrusive/src/eviction/lfu.rs | 2 +-
 foyer-storage/src/generic.rs        | 2 +-
 foyer-workspace-hack/Cargo.toml     | 8 +++-----
 scripts/install-deps.sh             | 2 +-
 10 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 3c6ecdd2..da602fd1 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -14,6 +14,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Run typos check
+        uses: crate-ci/typos@master
       - name: Install yq
         run: |
           wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0583a71c..5a00ccb5 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -21,6 +21,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Run typos check
+        uses: crate-ci/typos@master
       - name: Install yq
         run: |
           wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 911c379f..bbfbcdc4 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -20,6 +20,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Run typos check
+        uses: crate-ci/typos@master
       - name: Install yq
         run: |
           wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4b69ea84..6acb5bf0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -59,7 +59,7 @@ The write model and the design of storage engine has been switched from CacheLib
 
 Introduces `Store`, `RuntimeStore`, `LazyStore` to simplify usage. In most cases, `RuntimeStore` is preferred to use a dedicated tokio runtime to serve **foyer** to avoid the influence to the user's runtime. If lazy-load is needed, use `RuntimeLazyStore` instead.
 
-The implementation of **foyer** is separated into multiple crates. But importing `foyer` is enought for it re-exports the crates that **foyer**'s user needs.
+The implementation of **foyer** is separated into multiple crates. But importing `foyer` is enough for it re-exports the crates that **foyer**'s user needs.
 
 Brief description about the subcrates:
 
diff --git a/Makefile b/Makefile
index 209fc389..fd9cfa87 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,7 @@ deps:
 	./scripts/install-deps.sh
 
 check:
+	typos
 	shellcheck ./scripts/*
 	./.github/template/generate.sh
 	./scripts/minimize-dashboards.sh
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 07469f2b..e245506f 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -40,7 +40,7 @@ pub trait Link: Send + Sync + 'static + Default + Debug {
 ///
 /// Pointer operations MUST be valid.
 ///
-/// [`Adapter`] is recommanded to be generated by macro `instrusive_adapter!`.
+/// [`Adapter`] is recommended to be generated by macro `instrusive_adapter!`.
 pub unsafe trait Adapter: Send + Sync + Debug + 'static {
     type Pointer: Pointer;
     type Link: Link;
@@ -62,7 +62,7 @@ pub unsafe trait Adapter: Send + Sync + Debug + 'static {
 ///
 /// Pointer operations MUST be valid.
 ///
-/// [`KeyAdapter`] is recommanded to be generated by macro `key_adapter!`.
+/// [`KeyAdapter`] is recommended to be generated by macro `key_adapter!`.
 pub unsafe trait KeyAdapter: Adapter {
     type Key: Key;
 
@@ -76,7 +76,7 @@ pub unsafe trait KeyAdapter: Adapter {
 ///
 /// Pointer operations MUST be valid.
 ///
-/// [`PriorityAdapter`] is recommanded to be generated by macro `priority_adapter!`.
+/// [`PriorityAdapter`] is recommended to be generated by macro `priority_adapter!`.
 pub unsafe trait PriorityAdapter: Adapter {
     type Priority: PartialEq + Eq + PartialOrd + Ord + Clone + Copy + Into<usize>;
 
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 6b1d34d8..ed478a65 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -139,7 +139,7 @@ where
     /// the window length counter
     window_size: usize,
 
-    /// maxumum value of window length which when hit the counters are halved
+    /// maximum value of window length which when hit the counters are halved
     max_window_size: usize,
 
     /// the capacity for which the counters are sized
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 745a1914..3864e39c 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -830,7 +830,7 @@ impl EntryHeader {
 ///
 /// # Safety
 ///
-/// `buf.len()` must excatly fit entry size
+/// `buf.len()` must exactly fit entry size
 fn read_entry<K, V>(buf: &[u8]) -> Result<(K, V)>
 where
     K: Key,
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 41e1ac8f..e5a6b76a 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -25,20 +25,18 @@ futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
 libc = { version = "0.2", features = ["extra_traits"] }
-log = { version = "0.4", default-features = false, features = ["std"] }
 memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
-socket2 = { version = "0.5", default-features = false, features = ["all"] }
 tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
-tokio-util = { version = "0.7", features = ["codec", "io"] }
-tower = { version = "0.4", features = ["balance", "buffer", "limit", "timeout", "util"] }
-tracing = { version = "0.1", features = ["log"] }
 tracing-core = { version = "0.1" }
 
 [build-dependencies]
 cc = { version = "1", default-features = false, features = ["parallel"] }
 either = { version = "1", default-features = false, features = ["use_std"] }
+proc-macro2 = { version = "1" }
+quote = { version = "1" }
+syn = { version = "2", features = ["extra-traits", "full", "visit-mut"] }
 
 ### END HAKARI SECTION
diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh
index bf0869ae..31641bc4 100755
--- a/scripts/install-deps.sh
+++ b/scripts/install-deps.sh
@@ -4,4 +4,4 @@ if [ -z "$(which cargo-binstall)" ]; then
     curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 fi
 
-cargo binstall -y cargo-hakari cargo-sort cargo-nextest
\ No newline at end of file
+cargo binstall -y cargo-hakari cargo-sort cargo-nextest typos-cli
\ No newline at end of file

From 8e1da89d88ebdb8cc410d06a49cc589b3e3a4011 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 21 Dec 2023 18:19:49 +0800
Subject: [PATCH 197/261] fix: fix benchmark tool read/write range sync (#247)

---
 foyer-storage-bench/src/main.rs | 97 ++++++++++++++++-----------------
 1 file changed, 48 insertions(+), 49 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 5b488202..a29d8b25 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -370,6 +370,17 @@ where
     }
 }
 
+struct Context {
+    w_rate: Option<f64>,
+    r_rate: Option<f64>,
+    w_index: AtomicU64,
+    r_index: AtomicU64,
+    entry_size_range: Range<usize>,
+    lookup_range: u64,
+    time: u64,
+    metrics: Metrics,
+}
+
 fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
 #[cfg(feature = "tokio-console")]
@@ -622,33 +633,22 @@ async fn bench(
         Some(args.r_rate * 1024.0 * 1024.0)
     };
 
-    let index = Arc::new(AtomicU64::new(0));
+    let context = Arc::new(Context {
+        w_rate,
+        r_rate,
+        w_index: AtomicU64::new(0),
+        r_index: AtomicU64::new(0),
+        entry_size_range: args.entry_size_min..args.entry_size_max + 1,
+        time: args.time,
+        metrics: metrics.clone(),
+        lookup_range: args.lookup_range,
+    });
 
     let w_handles = (0..args.writers)
-        .map(|_| {
-            tokio::spawn(write(
-                args.entry_size_min..args.entry_size_max + 1,
-                w_rate,
-                index.clone(),
-                store.clone(),
-                args.time,
-                metrics.clone(),
-                stop_tx.subscribe(),
-            ))
-        })
+        .map(|_| tokio::spawn(write(store.clone(), context.clone(), stop_tx.subscribe())))
         .collect_vec();
     let r_handles = (0..args.readers)
-        .map(|_| {
-            tokio::spawn(read(
-                r_rate,
-                index.clone(),
-                store.clone(),
-                args.time,
-                metrics.clone(),
-                stop_tx.subscribe(),
-                args.lookup_range,
-            ))
-        })
+        .map(|_| tokio::spawn(read(store.clone(), context.clone(), stop_tx.subscribe())))
         .collect_vec();
 
     join_all(w_handles).await;
@@ -656,30 +656,26 @@ async fn bench(
 }
 
 async fn write(
-    entry_size_range: Range<usize>,
-    rate: Option<f64>,
-    index: Arc<AtomicU64>,
     store: impl Storage<Key = u64, Value = Arc<Vec<u8>>>,
-    time: u64,
-    metrics: Metrics,
+    context: Arc<Context>,
     mut stop: broadcast::Receiver<()>,
 ) {
     let start = Instant::now();
 
-    let mut limiter = rate.map(RateLimiter::new);
+    let mut limiter = context.w_rate.map(RateLimiter::new);
 
     loop {
         match stop.try_recv() {
             Err(broadcast::error::TryRecvError::Empty) => {}
             _ => return,
         }
-        if start.elapsed().as_secs() >= time {
+        if start.elapsed().as_secs() >= context.time {
             return;
         }
 
-        let idx = index.fetch_add(1, Ordering::Relaxed);
+        let idx = context.w_index.fetch_add(1, Ordering::Relaxed);
         // TODO(MrCroxx): Use random content?
-        let entry_size = OsRng.gen_range(entry_size_range.clone());
+        let entry_size = OsRng.gen_range(context.entry_size_range.clone());
         let data = Arc::new(text(idx as usize, entry_size));
         if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
             tokio::time::sleep(wait).await;
@@ -688,13 +684,15 @@ async fn write(
         let time = Instant::now();
         let inserted = store.insert(idx, data).await.unwrap();
         let lat = time.elapsed().as_micros() as u64;
-        if let Err(e) = metrics.insert_lats.write().record(lat) {
+        context.r_index.fetch_add(1, Ordering::Relaxed);
+        if let Err(e) = context.metrics.insert_lats.write().record(lat) {
             tracing::error!("metrics error: {:?}, value: {}", e, lat);
         }
 
         if inserted {
-            metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
-            metrics
+            context.metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
+            context
+                .metrics
                 .insert_bytes
                 .fetch_add(entry_size, Ordering::Relaxed);
         }
@@ -702,17 +700,13 @@ async fn write(
 }
 
 async fn read(
-    rate: Option<f64>,
-    index: Arc<AtomicU64>,
     store: impl Storage<Key = u64, Value = Arc<Vec<u8>>>,
-    time: u64,
-    metrics: Metrics,
+    context: Arc<Context>,
     mut stop: broadcast::Receiver<()>,
-    look_up_range: u64,
 ) {
     let start = Instant::now();
 
-    let mut limiter = rate.map(RateLimiter::new);
+    let mut limiter = context.r_rate.map(RateLimiter::new);
 
     let mut rng = StdRng::seed_from_u64(0);
 
@@ -721,12 +715,14 @@ async fn read(
             Err(broadcast::error::TryRecvError::Empty) => {}
             _ => return,
         }
-        if start.elapsed().as_secs() >= time {
+        if start.elapsed().as_secs() >= context.time {
             return;
         }
 
-        let idx_max = index.load(Ordering::Relaxed);
-        let idx = rng.gen_range(std::cmp::max(idx_max, look_up_range) - look_up_range..=idx_max);
+        let idx_max = context.r_index.load(Ordering::Relaxed);
+        let idx = rng.gen_range(
+            std::cmp::max(idx_max, context.lookup_range) - context.lookup_range..=idx_max,
+        );
 
         let time = Instant::now();
         let res = store.lookup(&idx).await.unwrap();
@@ -735,21 +731,24 @@ async fn read(
         if let Some(buf) = res {
             let entry_size = buf.len();
             assert_eq!(&text(idx as usize, entry_size), buf.as_ref());
-            if let Err(e) = metrics.get_hit_lats.write().record(lat) {
+            if let Err(e) = context.metrics.get_hit_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);
             }
-            metrics.get_bytes.fetch_add(entry_size, Ordering::Relaxed);
+            context
+                .metrics
+                .get_bytes
+                .fetch_add(entry_size, Ordering::Relaxed);
 
             if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
                 tokio::time::sleep(wait).await;
             }
         } else {
-            if let Err(e) = metrics.get_miss_lats.write().record(lat) {
+            if let Err(e) = context.metrics.get_miss_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);
             }
-            metrics.get_miss_ios.fetch_add(1, Ordering::Relaxed);
+            context.metrics.get_miss_ios.fetch_add(1, Ordering::Relaxed);
         }
-        metrics.get_ios.fetch_add(1, Ordering::Relaxed);
+        context.metrics.get_ios.fetch_add(1, Ordering::Relaxed);
 
         tokio::task::consume_budget().await;
     }

From aac74bfbdef0c8321549450d8efa97dadfd1b3ae Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 21 Dec 2023 23:22:06 +0800
Subject: [PATCH 198/261] fix: fix another factor that make bench miss ratio
 not accurate (#249)

---
 foyer-storage-bench/src/main.rs | 46 +++++++++++++++++++++++++--------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index a29d8b25..c3daf183 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -373,8 +373,7 @@ where
 struct Context {
     w_rate: Option<f64>,
     r_rate: Option<f64>,
-    w_index: AtomicU64,
-    r_index: AtomicU64,
+    counts: Vec<AtomicU64>,
     entry_size_range: Range<usize>,
     lookup_range: u64,
     time: u64,
@@ -477,6 +476,11 @@ async fn main() {
 
     println!("{:#?}", args);
 
+    assert!(
+        args.lookup_range > 0,
+        "\"--lookup-range\" value must be greater than 0"
+    );
+
     create_dir_all(&args.dir).unwrap();
 
     let iostat_path = match detect_fs_type(&args.dir) {
@@ -633,11 +637,14 @@ async fn bench(
         Some(args.r_rate * 1024.0 * 1024.0)
     };
 
+    let counts = (0..args.writers)
+        .map(|_| AtomicU64::default())
+        .collect_vec();
+
     let context = Arc::new(Context {
         w_rate,
         r_rate,
-        w_index: AtomicU64::new(0),
-        r_index: AtomicU64::new(0),
+        counts,
         entry_size_range: args.entry_size_min..args.entry_size_max + 1,
         time: args.time,
         metrics: metrics.clone(),
@@ -645,7 +652,14 @@ async fn bench(
     });
 
     let w_handles = (0..args.writers)
-        .map(|_| tokio::spawn(write(store.clone(), context.clone(), stop_tx.subscribe())))
+        .map(|id| {
+            tokio::spawn(write(
+                id as u64,
+                store.clone(),
+                context.clone(),
+                stop_tx.subscribe(),
+            ))
+        })
         .collect_vec();
     let r_handles = (0..args.readers)
         .map(|_| tokio::spawn(read(store.clone(), context.clone(), stop_tx.subscribe())))
@@ -656,6 +670,7 @@ async fn bench(
 }
 
 async fn write(
+    id: u64,
     store: impl Storage<Key = u64, Value = Arc<Vec<u8>>>,
     context: Arc<Context>,
     mut stop: broadcast::Receiver<()>,
@@ -663,6 +678,8 @@ async fn write(
     let start = Instant::now();
 
     let mut limiter = context.w_rate.map(RateLimiter::new);
+    let step = context.counts.len() as u64;
+    let count = &context.counts[id as usize];
 
     loop {
         match stop.try_recv() {
@@ -673,7 +690,8 @@ async fn write(
             return;
         }
 
-        let idx = context.w_index.fetch_add(1, Ordering::Relaxed);
+        let c = count.load(Ordering::Relaxed);
+        let idx = id + step * c;
         // TODO(MrCroxx): Use random content?
         let entry_size = OsRng.gen_range(context.entry_size_range.clone());
         let data = Arc::new(text(idx as usize, entry_size));
@@ -684,7 +702,7 @@ async fn write(
         let time = Instant::now();
         let inserted = store.insert(idx, data).await.unwrap();
         let lat = time.elapsed().as_micros() as u64;
-        context.r_index.fetch_add(1, Ordering::Relaxed);
+        count.store(c + 1, Ordering::Relaxed);
         if let Err(e) = context.metrics.insert_lats.write().record(lat) {
             tracing::error!("metrics error: {:?}, value: {}", e, lat);
         }
@@ -707,6 +725,7 @@ async fn read(
     let start = Instant::now();
 
     let mut limiter = context.r_rate.map(RateLimiter::new);
+    let step = context.counts.len() as u64;
 
     let mut rng = StdRng::seed_from_u64(0);
 
@@ -719,10 +738,15 @@ async fn read(
             return;
         }
 
-        let idx_max = context.r_index.load(Ordering::Relaxed);
-        let idx = rng.gen_range(
-            std::cmp::max(idx_max, context.lookup_range) - context.lookup_range..=idx_max,
-        );
+        let w = rng.gen_range(0..step); // pick a writer to read form
+        let c_max = context.counts[w as usize].load(Ordering::Relaxed);
+        if c_max == 0 {
+            tokio::time::sleep(Duration::from_millis(1)).await;
+            continue;
+        }
+        let c =
+            rng.gen_range(std::cmp::max(c_max, context.lookup_range) - context.lookup_range..c_max);
+        let idx = w + c * step;
 
         let time = Instant::now();
         let res = store.lookup(&idx).await.unwrap();

From d9a78e09530a61c7e7747d47f87d8188c9830c33 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 22 Dec 2023 15:02:43 +0800
Subject: [PATCH 199/261] chore: remove flusher_buffer_capacity to simplify
 config (#250)

---
 foyer-storage-bench/src/main.rs               | 4 ----
 foyer-storage/src/admission/rated_ticket.rs   | 4 ----
 foyer-storage/src/buffer.rs                   | 8 ++++----
 foyer-storage/src/flusher.rs                  | 3 +--
 foyer-storage/src/generic.rs                  | 8 --------
 foyer-storage/src/lazy.rs                     | 2 --
 foyer-storage/src/reinsertion/exist.rs        | 7 +------
 foyer-storage/src/reinsertion/rated_ticket.rs | 4 ----
 foyer-storage/src/storage.rs                  | 1 -
 foyer-storage/tests/storage_test.rs           | 6 ------
 10 files changed, 6 insertions(+), 41 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index c3daf183..f41bb7fc 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -102,9 +102,6 @@ pub struct Args {
     #[arg(long, default_value_t = 64)]
     region_size: usize,
 
-    #[arg(long, default_value_t = 0)]
-    flusher_buffer_size: usize,
-
     #[arg(long, default_value_t = 4)]
     flushers: usize,
 
@@ -544,7 +541,6 @@ async fn main() {
         catalog_bits: args.catalog_bits,
         admissions,
         reinsertions,
-        flusher_buffer_size: args.flusher_buffer_size,
         flushers: args.flushers,
         reclaimers: args.reclaimers,
         recover_concurrency: args.recover_concurrency,
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
index cc6318af..c886571e 100644
--- a/foyer-storage/src/admission/rated_ticket.rs
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -14,7 +14,6 @@
 
 use std::{
     fmt::Debug,
-    marker::PhantomData,
     sync::{
         atomic::{AtomicUsize, Ordering},
         OnceLock,
@@ -39,8 +38,6 @@ where
     last: AtomicUsize,
 
     context: OnceLock<AdmissionContext<K, V>>,
-
-    _marker: PhantomData<(K, V)>,
 }
 
 impl<K, V> RatedTicketAdmissionPolicy<K, V>
@@ -53,7 +50,6 @@ where
             inner: RatedTicket::new(rate as f64),
             last: AtomicUsize::default(),
             context: OnceLock::new(),
-            _marker: PhantomData,
         }
     }
 }
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index c191ad89..48d88b3e 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -86,8 +86,9 @@ where
     V: Value,
     D: Device,
 {
-    pub fn new(device: D, default_buffer_capacity: usize) -> Self {
-        let default_buffer_capacity = std::cmp::max(default_buffer_capacity, device.io_size());
+    pub fn new(device: D) -> Self {
+        let default_buffer_capacity =
+            align_up(device.align(), device.io_size() + device.io_size() / 2);
         let buffer = device.io_buffer(0, default_buffer_capacity);
         Self {
             buffer,
@@ -343,9 +344,8 @@ mod tests {
         })
         .await
         .unwrap();
-        const DEFAULT_BUFFER_CAPACITY: usize = 32 * 1024;
 
-        let mut buffer = FlushBuffer::new(device.clone(), DEFAULT_BUFFER_CAPACITY);
+        let mut buffer = FlushBuffer::new(device.clone());
         assert_eq!(buffer.region(), None);
 
         const HEADER: usize = EntryHeader::serialized_len();
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 8b71d8b5..333b5023 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -99,7 +99,6 @@ where
     EL: Link,
 {
     pub fn new(
-        default_buffer_capacity: usize,
         region_manager: Arc<RegionManager<D, EP, EL>>,
         catalog: Arc<Catalog<K, V>>,
         device: D,
@@ -107,7 +106,7 @@ where
         metrics: Arc<Metrics>,
         stop_rx: broadcast::Receiver<()>,
     ) -> Self {
-        let buffer = FlushBuffer::new(device.clone(), default_buffer_capacity);
+        let buffer = FlushBuffer::new(device.clone());
         Self {
             region_manager,
             catalog,
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 3864e39c..f40cb6a7 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -85,9 +85,6 @@ where
     /// Reinsertion policies.
     pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
 
-    /// Flusher default buffer capacity, must be equals or larger than device io size.
-    pub flusher_buffer_size: usize,
-
     /// Count of flushers.
     pub flushers: usize,
 
@@ -120,7 +117,6 @@ where
             .field("catalog_bits", &self.catalog_bits)
             .field("admissions", &self.admissions)
             .field("reinsertions", &self.reinsertions)
-            .field("flusher_buffer_size", &self.flusher_buffer_size)
             .field("flushers", &self.flushers)
             .field("reclaimers", &self.reclaimers)
             .field("clean_region_threshold", &self.clean_region_threshold)
@@ -145,7 +141,6 @@ where
             catalog_bits: self.catalog_bits,
             admissions: self.admissions.clone(),
             reinsertions: self.reinsertions.clone(),
-            flusher_buffer_size: self.flusher_buffer_size,
             flushers: self.flushers,
             reclaimers: self.reclaimers,
             clean_region_threshold: self.clean_region_threshold,
@@ -301,7 +296,6 @@ where
             .zip_eq(flusher_entry_rxs.into_iter())
             .map(|(stop_rx, entry_rx)| {
                 Flusher::new(
-                    config.flusher_buffer_size,
                     region_manager.clone(),
                     catalog.clone(),
                     device.clone(),
@@ -1154,7 +1148,6 @@ mod tests {
             catalog_bits: 1,
             admissions,
             reinsertions,
-            flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
             recover_concurrency: 2,
@@ -1204,7 +1197,6 @@ mod tests {
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
-            flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 0,
             recover_concurrency: 2,
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 26c08246..1b4b6e3e 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -256,7 +256,6 @@ mod tests {
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
-            flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
             recover_concurrency: 2,
@@ -289,7 +288,6 @@ mod tests {
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
-            flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
             recover_concurrency: 2,
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index 69718000..2dda6a31 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -12,10 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    marker::PhantomData,
-    sync::{Arc, OnceLock},
-};
+use std::sync::{Arc, OnceLock};
 
 use foyer_common::code::{Key, Value};
 
@@ -29,7 +26,6 @@ where
     V: Value,
 {
     catalog: OnceLock<Arc<Catalog<K, V>>>,
-    _marker: PhantomData<V>,
 }
 
 impl<K, V> Default for ExistReinsertionPolicy<K, V>
@@ -40,7 +36,6 @@ where
     fn default() -> Self {
         Self {
             catalog: OnceLock::new(),
-            _marker: PhantomData,
         }
     }
 }
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
index d96ef806..46c06e74 100644
--- a/foyer-storage/src/reinsertion/rated_ticket.rs
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -14,7 +14,6 @@
 
 use std::{
     fmt::Debug,
-    marker::PhantomData,
     sync::{
         atomic::{AtomicUsize, Ordering},
         OnceLock,
@@ -39,8 +38,6 @@ where
     last: AtomicUsize,
 
     context: OnceLock<ReinsertionContext<K, V>>,
-
-    _marker: PhantomData<(K, V)>,
 }
 
 impl<K, V> RatedTicketReinsertionPolicy<K, V>
@@ -53,7 +50,6 @@ where
             inner: RatedTicket::new(rate as f64),
             last: AtomicUsize::default(),
             context: OnceLock::new(),
-            _marker: PhantomData,
         }
     }
 }
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 9ae396de..074306be 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -370,7 +370,6 @@ mod tests {
             catalog_bits: 1,
             admissions: vec![],
             reinsertions: vec![],
-            flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
             clean_region_threshold: 1,
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index dae305e5..46459f32 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -133,7 +133,6 @@ async fn test_store() {
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
-        flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
         clean_region_threshold: 1,
@@ -161,7 +160,6 @@ async fn test_store_zstd() {
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
-        flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
         clean_region_threshold: 1,
@@ -189,7 +187,6 @@ async fn test_store_lz4() {
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
-        flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
         clean_region_threshold: 1,
@@ -217,7 +214,6 @@ async fn test_lazy_store() {
         catalog_bits: 1,
         admissions: vec![recorder.clone()],
         reinsertions: vec![recorder.clone()],
-        flusher_buffer_size: 0,
         flushers: 1,
         reclaimers: 1,
         clean_region_threshold: 1,
@@ -246,7 +242,6 @@ async fn test_runtime_store() {
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
-            flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
             clean_region_threshold: 1,
@@ -281,7 +276,6 @@ async fn test_runtime_lazy_store() {
             catalog_bits: 1,
             admissions: vec![recorder.clone()],
             reinsertions: vec![recorder.clone()],
-            flusher_buffer_size: 0,
             flushers: 1,
             reclaimers: 1,
             clean_region_threshold: 1,

From aca33e9f7df52157b2b57e921ebc6fdb7dea2034 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 22 Dec 2023 15:16:57 +0800
Subject: [PATCH 200/261] chore: bump foyer version (#251)

---
 CHANGELOG.md                    | 21 +++++++++++++++++++++
 foyer-storage-bench/Cargo.toml  |  4 ++--
 foyer-storage/Cargo.toml        |  2 +-
 foyer-workspace-hack/Cargo.toml |  2 +-
 foyer/Cargo.toml                |  4 ++--
 5 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6acb5bf0..33e3af7a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,24 @@
+## 2023-12-22
+
+| crate | version |
+| - | - |
+| foyer | 0.4.0 |
+| foyer-storage | 0.3.0 |
+| foyer-storage-bench | 0.3.0 |
+| foyer-workspace-hack | 0.1.1 |
+
+<details>
+
+### Changes
+
+- Remove config `flusher_buffer_capacity`.
+
+### Fixes
+
+- Fix benchmark tool cache miss ratio.
+
+</details>
+
 ## 2023-12-20
 
 | crate | version |
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index b52333ff..c00cf56d 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage-bench"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine bench tool for foyer - the hybrid cache for Rust"
@@ -19,7 +19,7 @@ clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
 foyer-common = { version = "0.2", path = "../foyer-common" }
 foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.2", path = "../foyer-storage" }
+foyer-storage = { version = "0.3", path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 6764398c..fb5ddd11 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.2.2"
+version = "0.3.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index e5a6b76a..a1c46186 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -4,7 +4,7 @@
 
 [package]
 name = "foyer-workspace-hack"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "workspace-hack package, managed by hakari"
 license = "Apache-2.0"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 66af894f..4e710255 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -15,5 +15,5 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 foyer-common = { version = "0.2", path = "../foyer-common" }
 foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.2", path = "../foyer-storage" }
+foyer-storage = { version = "0.3", path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }

From d3b375ee7e2a4cc6b0c66243ec973a519645b3b8 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 22 Dec 2023 16:08:43 +0800
Subject: [PATCH 201/261] fix: fix duplicated insert dropped metrics (#252)

* fix: fix duplicated insert dropped metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: tiny refactor

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/generic.rs | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index f40cb6a7..094dbcfa 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -735,16 +735,9 @@ where
         if !self.is_inserted {
             debug_assert!(self.key.is_some());
 
-            self.store
-                .inner
-                .metrics
-                .op_duration_insert_dropped
-                .observe(self.duration.as_secs_f64());
-
+            let filtered = self.is_judged && !self.judge();
             // make sure each key after `judge` will call either `on_insert` or `on_drop`.
-            let mut filtered = false;
             if self.is_judged {
-                filtered = !self.judge();
                 for (i, admission) in self.store.inner.admissions.iter().enumerate() {
                     let judge = self.judges.get(i);
                     admission.on_drop(self.key.as_ref().unwrap(), self.weight, judge);

From 32d5d3d7d1483121276e3bae54707e8cae187b1f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 26 Dec 2023 16:55:07 +0800
Subject: [PATCH 202/261] feat: introduce time-series distribution args to
 bench tool (#253)

---
 foyer-storage-bench/Cargo.toml  |   1 +
 foyer-storage-bench/src/main.rs | 217 +++++++++++++++++++++++++++++---
 foyer-storage/src/flusher.rs    |   4 +
 3 files changed, 206 insertions(+), 16 deletions(-)

diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index c00cf56d..d8ad36d1 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -49,6 +49,7 @@ tokio = { workspace = true }
 tracing = "0.1"
 tracing-opentelemetry = { version = "0.22", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+zipf = "7"
 
 [features]
 deadlock = ["parking_lot/deadlock_detection", "foyer-storage/deadlock"]
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index f41bb7fc..218487fa 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -22,6 +22,7 @@ mod text;
 mod utils;
 
 use std::{
+    collections::BTreeMap,
     fs::create_dir_all,
     ops::Range,
     path::PathBuf,
@@ -44,12 +45,13 @@ use foyer_storage::{
     error::Result,
     reinsertion::{rated_ticket::RatedTicketReinsertionPolicy, ReinsertionPolicy},
     runtime::{RuntimeConfig, RuntimeStore, RuntimeStoreConfig, RuntimeStoreWriter},
-    storage::{Storage, StorageExt, StorageWriter},
+    storage::{AsyncStorageExt, Storage, StorageExt, StorageWriter},
     store::{LfuFsStoreConfig, Store, StoreConfig, StoreWriter},
 };
 use futures::future::join_all;
 use itertools::Itertools;
 use rand::{
+    distributions::Distribution,
     rngs::{OsRng, StdRng},
     Rng, SeedableRng,
 };
@@ -59,6 +61,7 @@ use tokio::sync::broadcast;
 use utils::{detect_fs_type, dev_stat_path, file_stat_path, iostat, FsType};
 
 #[derive(Parser, Debug, Clone)]
+#[command(author, version, about)]
 pub struct Args {
     /// dir for cache data
     #[arg(short, long)]
@@ -152,6 +155,22 @@ pub struct Args {
     /// available values: "none", "zstd"
     #[arg(long, default_value = "none")]
     compression: String,
+
+    /// Time-series operation distribution.
+    ///
+    /// Available values: "none", "uniform", "zipf".
+    ///
+    /// If "uniform" or "zipf" is used, operations will be performed in async mode.
+    #[arg(long, default_value = "none")]
+    distribution: String,
+
+    /// For `--distribution zipf` only.
+    #[arg(long, default_value_t = 100)]
+    distribution_zipf_n: usize,
+
+    /// For `--distribution zipf` only.
+    #[arg(long, default_value_t = 0.5)]
+    distribution_zipf_s: f64,
 }
 
 #[derive(Debug)]
@@ -367,6 +386,47 @@ where
     }
 }
 
+#[derive(Debug)]
+enum TimeSeriesDistribution {
+    None,
+    Uniform {
+        interval: Duration,
+    },
+    Zipf {
+        n: usize,
+        s: f64,
+        interval: Duration,
+    },
+}
+
+impl TimeSeriesDistribution {
+    fn new(args: &Args) -> Self {
+        match args.distribution.as_str() {
+            "none" => TimeSeriesDistribution::None,
+            "uniform" => {
+                // interval = 1 / freq = 1 / (rate / size) = size / rate
+                let interval = ((args.entry_size_min + args.entry_size_max) >> 1) as f64
+                    / (args.w_rate * 1024.0 * 1024.0);
+                let interval = Duration::from_secs_f64(interval);
+                TimeSeriesDistribution::Uniform { interval }
+            }
+            "zipf" => {
+                // interval = 1 / freq = 1 / (rate / size) = size / rate
+                let interval = ((args.entry_size_min + args.entry_size_max) >> 1) as f64
+                    / (args.w_rate * 1024.0 * 1024.0);
+                let interval = Duration::from_secs_f64(interval);
+                display_zipf_sample(args.distribution_zipf_n, args.distribution_zipf_s);
+                TimeSeriesDistribution::Zipf {
+                    n: args.distribution_zipf_n,
+                    s: args.distribution_zipf_s,
+                    interval,
+                }
+            }
+            other => panic!("unsupported distribution: {}", other),
+        }
+    }
+}
+
 struct Context {
     w_rate: Option<f64>,
     r_rate: Option<f64>,
@@ -374,6 +434,7 @@ struct Context {
     entry_size_range: Range<usize>,
     lookup_range: u64,
     time: u64,
+    distribution: TimeSeriesDistribution,
     metrics: Metrics,
 }
 
@@ -637,14 +698,17 @@ async fn bench(
         .map(|_| AtomicU64::default())
         .collect_vec();
 
+    let distribution = TimeSeriesDistribution::new(&args);
+
     let context = Arc::new(Context {
         w_rate,
         r_rate,
+        lookup_range: args.lookup_range,
         counts,
         entry_size_range: args.entry_size_min..args.entry_size_max + 1,
         time: args.time,
+        distribution,
         metrics: metrics.clone(),
-        lookup_range: args.lookup_range,
     });
 
     let w_handles = (0..args.writers)
@@ -675,9 +739,47 @@ async fn write(
 
     let mut limiter = context.w_rate.map(RateLimiter::new);
     let step = context.counts.len() as u64;
-    let count = &context.counts[id as usize];
+
+    const K: usize = 100;
+    const G: usize = 10;
+
+    let zipf_intervals = match context.distribution {
+        TimeSeriesDistribution::Zipf { n, s, interval } => {
+            let histogram = gen_zipf_histogram(n, s, G, n * K);
+
+            let loop_interval = Duration::from_secs_f64(interval.as_secs_f64() * K as f64);
+            let group_cnt = K / G;
+            let group_interval = interval.as_secs_f64() * group_cnt as f64;
+
+            if id == 0 {
+                println!("loop interval: {loop_interval:?}, zipf intervals: ");
+            }
+
+            let mut sum = 0;
+            let intervals = histogram
+                .values()
+                .copied()
+                .map(|ratio| {
+                    let cnt = ratio * K as f64;
+                    sum += cnt as usize;
+                    let interval = Duration::from_secs_f64(group_interval / cnt);
+                    if id == 0 {
+                        println!("    [{cnt:3.0} ==> {interval:010.3?}]");
+                    }
+                    (sum, interval)
+                })
+                .collect_vec();
+
+            Some(intervals)
+        }
+        _ => None,
+    };
+
+    let mut c = 0;
 
     loop {
+        let l = Instant::now();
+
         match stop.try_recv() {
             Err(broadcast::error::TryRecvError::Empty) => {}
             _ => return,
@@ -686,7 +788,6 @@ async fn write(
             return;
         }
 
-        let c = count.load(Ordering::Relaxed);
         let idx = id + step * c;
         // TODO(MrCroxx): Use random content?
         let entry_size = OsRng.gen_range(context.entry_size_range.clone());
@@ -696,20 +797,49 @@ async fn write(
         }
 
         let time = Instant::now();
-        let inserted = store.insert(idx, data).await.unwrap();
-        let lat = time.elapsed().as_micros() as u64;
-        count.store(c + 1, Ordering::Relaxed);
-        if let Err(e) = context.metrics.insert_lats.write().record(lat) {
-            tracing::error!("metrics error: {:?}, value: {}", e, lat);
-        }
+        let ctx = context.clone();
+        let callback = move |res: Result<bool>| async move {
+            let inserted = res.unwrap();
+            let lat = time.elapsed().as_micros() as u64;
+            ctx.counts[id as usize].fetch_add(1, Ordering::Relaxed);
+            if let Err(e) = ctx.metrics.insert_lats.write().record(lat) {
+                tracing::error!("metrics error: {:?}, value: {}", e, lat);
+            }
 
-        if inserted {
-            context.metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
-            context
-                .metrics
-                .insert_bytes
-                .fetch_add(entry_size, Ordering::Relaxed);
+            if inserted {
+                ctx.metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
+                ctx.metrics
+                    .insert_bytes
+                    .fetch_add(entry_size, Ordering::Relaxed);
+            }
+        };
+
+        let elapsed = l.elapsed();
+
+        match &context.distribution {
+            TimeSeriesDistribution::None => {
+                let res = store.insert(idx, data).await;
+                callback(res).await;
+            }
+            TimeSeriesDistribution::Uniform { interval } => {
+                store.insert_async_with_callback(idx, data, callback);
+                tokio::time::sleep(interval.saturating_sub(elapsed)).await;
+            }
+            TimeSeriesDistribution::Zipf { .. } => {
+                store.insert_async_with_callback(idx, data, callback);
+                let intervals = zipf_intervals.as_ref().unwrap();
+
+                let group = match intervals.binary_search_by_key(&(c as usize % K), |(sum, _)| *sum)
+                {
+                    Ok(i) => i,
+                    Err(i) => i.min(G - 1),
+                };
+
+                tokio::time::sleep(intervals[group].1.saturating_sub(elapsed)).await;
+            }
         }
+
+        c += 1;
     }
 }
 
@@ -773,3 +903,58 @@ async fn read(
         tokio::task::consume_budget().await;
     }
 }
+
+fn gen_zipf_histogram(n: usize, s: f64, groups: usize, samples: usize) -> BTreeMap<usize, f64> {
+    let step = n / groups;
+
+    let mut rng = rand::thread_rng();
+    let zipf = zipf::ZipfDistribution::new(n, s).unwrap();
+    let mut data: BTreeMap<usize, usize> = BTreeMap::default();
+    for _ in 0..samples {
+        let v = zipf.sample(&mut rng);
+        let g = std::cmp::min(v / step, groups);
+        *data.entry(g).or_default() += 1;
+    }
+    let mut histogram: BTreeMap<usize, f64> = BTreeMap::default();
+    for group in 0..groups {
+        histogram.insert(
+            group,
+            data.get(&group).copied().unwrap_or_default() as f64 / samples as f64,
+        );
+    }
+    histogram
+}
+
+fn display_zipf_sample(n: usize, s: f64) {
+    const W: usize = 100;
+    const H: usize = 10;
+
+    let samples = n * 1000;
+
+    let histogram = gen_zipf_histogram(n, s, H, samples);
+
+    let max = histogram.values().copied().fold(0.0, f64::max);
+
+    println!("zipf's diagram [N = {n}][s = {s}][samples = {}]", n * 1000);
+
+    for (g, ratio) in histogram {
+        let shares = (ratio / max * W as f64) as usize;
+        let bar: String = if shares != 0 {
+            "=".repeat(shares)
+        } else {
+            ".".to_string()
+        };
+        println!(
+            "{:3} : {:6} : {:6.3}% : {}",
+            g,
+            (samples as f64 * ratio) as usize,
+            ratio * 100.0,
+            bar
+        );
+    }
+}
+
+#[test]
+fn zipf() {
+    display_zipf_sample(1000, 0.5);
+}
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 333b5023..0ad99c61 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -192,6 +192,10 @@ where
 
     #[tracing::instrument(skip(self))]
     async fn update_catalog(&self, entries: Vec<PositionedEntry<K, V>>) -> Result<()> {
+        if entries.is_empty() {
+            return Ok(());
+        }
+
         // record fully flushed bytes by the way
         let mut bytes = 0;
 

From fdde9728f5e57d8e7ed9e4859c3d3eabaebeb60c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 26 Dec 2023 17:23:48 +0800
Subject: [PATCH 203/261] chore: remove unused ut (#254)

---
 foyer-storage-bench/src/main.rs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 218487fa..92d9dfd5 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -953,8 +953,3 @@ fn display_zipf_sample(n: usize, s: f64) {
         );
     }
 }
-
-#[test]
-fn zipf() {
-    display_zipf_sample(1000, 0.5);
-}

From 5df49764539fb3ba8af23fa6966f5b74b25d0332 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 27 Dec 2023 18:05:04 +0800
Subject: [PATCH 204/261] chore: bump rust toolchain (#255)

---
 .github/template/template.yml      |  4 ++--
 .github/workflows/main.yml         |  4 ++--
 .github/workflows/pull-request.yml |  4 ++--
 foyer-intrusive/src/lib.rs         |  1 -
 foyer-storage-bench/src/main.rs    |  8 ++++++--
 foyer-storage/src/catalog.rs       | 12 +++++++++---
 rust-toolchain                     |  2 +-
 7 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index da602fd1..e320f870 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -3,9 +3,9 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-10-21
+  RUST_TOOLCHAIN: nightly-2023-12-26
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231010v3
+  CACHE_KEY_SUFFIX: 20231227v1
 
 jobs:
   misc-check:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5a00ccb5..fb854135 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,9 +11,9 @@ on:
     branches: [main]
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2023-10-21
+  RUST_TOOLCHAIN: nightly-2023-12-26
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231010v3
+  CACHE_KEY_SUFFIX: 20231227v1
 jobs:
   misc-check:
     name: misc check
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index bbfbcdc4..bf9cff0a 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -10,9 +10,9 @@ on:
   pull_request:
     branches: [main]
 env:
-  RUST_TOOLCHAIN: nightly-2023-10-21
+  RUST_TOOLCHAIN: nightly-2023-12-26
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231010v3
+  CACHE_KEY_SUFFIX: 20231227v1
 jobs:
   misc-check:
     name: misc check
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 8f93d2fd..a8126dac 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -17,7 +17,6 @@
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
 #![expect(clippy::new_without_default)]
-#![cfg_attr(test, expect(clippy::vtable_address_comparisons))]
 
 pub use memoffset::offset_of;
 
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 92d9dfd5..6c5f5566 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -792,7 +792,9 @@ async fn write(
         // TODO(MrCroxx): Use random content?
         let entry_size = OsRng.gen_range(context.entry_size_range.clone());
         let data = Arc::new(text(idx as usize, entry_size));
-        if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
+        if let Some(limiter) = &mut limiter
+            && let Some(wait) = limiter.consume(entry_size as f64)
+        {
             tokio::time::sleep(wait).await;
         }
 
@@ -889,7 +891,9 @@ async fn read(
                 .get_bytes
                 .fetch_add(entry_size, Ordering::Relaxed);
 
-            if let Some(limiter) = &mut limiter  && let Some(wait) = limiter.consume(entry_size as f64) {
+            if let Some(limiter) = &mut limiter
+                && let Some(wait) = limiter.consume(entry_size as f64)
+            {
                 tokio::time::sleep(wait).await;
             }
         } else {
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 7dd94267..358791ee 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -134,8 +134,12 @@ where
             item.inserted = Some(Instant::now());
             guard.insert(key.clone(), item)
         };
-        if let Some(old) = old && let Index::Inflight { .. } = old.index() {
-            self.metrics.inner_op_duration_entry_flush.observe(old.inserted.unwrap().elapsed().as_secs_f64());
+        if let Some(old) = old
+            && let Index::Inflight { .. } = old.index()
+        {
+            self.metrics
+                .inner_op_duration_entry_flush
+                .observe(old.inserted.unwrap().elapsed().as_secs_f64());
         }
     }
 
@@ -147,7 +151,9 @@ where
     pub fn remove(&self, key: &K) -> Option<Item<K, V>> {
         let shard = self.shard(key);
         let info: Option<Item<K, V>> = self.items[shard].write().remove(key);
-        if let Some(info) = &info && let Index::Region { view} = &info.index {
+        if let Some(info) = &info
+            && let Index::Region { view } = &info.index
+        {
             self.regions[*view.id() as usize].lock().remove(key);
         }
         info
diff --git a/rust-toolchain b/rust-toolchain
index fe2a026f..ef0ed508 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-10-21"
+channel = "nightly-2023-12-26"
\ No newline at end of file

From eef8d78bbd16eed147468885c4b48dcd2e120626 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 28 Dec 2023 14:42:40 +0800
Subject: [PATCH 205/261] chore: bump foyer 0.5.0 (#256)

---
 CHANGELOG.md                    | 24 ++++++++++++++++++++++++
 foyer-common/Cargo.toml         |  4 ++--
 foyer-intrusive/Cargo.toml      |  6 +++---
 foyer-memory/Cargo.toml         |  2 +-
 foyer-storage-bench/Cargo.toml  | 10 +++++-----
 foyer-storage/Cargo.toml        |  8 ++++----
 foyer-workspace-hack/Cargo.toml |  2 +-
 foyer/Cargo.toml                | 10 +++++-----
 8 files changed, 45 insertions(+), 21 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33e3af7a..6f3e5fb0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,27 @@
+## 2023-12-28
+
+| crate | version |
+| - | - |
+| foyer | 0.5.0 |
+| foyer-common | 0.3.0 |
+| foyer-intrusive | 0.2.0 |
+| foyer-storage | 0.4.0 |
+| foyer-storage-bench | 0.4.0 |
+| foyer-workspace-hack | 0.2.0 |
+
+<details>
+
+### Changes
+
+- Bump rust-toolchain to "nightly-2023-12-26".
+- Introduce time-series distribution args to bench tool. [#253](https://github.com/MrCroxx/foyer/pull/253)
+
+### Fixes
+
+- Fix duplicated insert drop metrics.
+
+</details>
+
 ## 2023-12-22
 
 | crate | version |
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 22d2e5c3..525a4393 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-common"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "common utils for foyer - the hybrid cache for Rust"
@@ -15,7 +15,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 anyhow = "1.0"
 bytes = "1"
-foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 35558b8e..87c89468 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-intrusive"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "intrusive data structures for foyer - the hybrid cache for Rust"
@@ -15,8 +15,8 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { version = "0.2", path = "../foyer-common" }
-foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.3", path = "../foyer-common" }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 memoffset = "0.9"
 parking_lot = "0.12"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 1ca7f47c..88c0b2fe 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -17,7 +17,7 @@ bytes = "1"
 cmsketch = "0.1"
 foyer-common = { path = "../foyer-common" }
 foyer-intrusive = { path = "../foyer-intrusive" }
-foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
 libc = "0.2"
 memoffset = "0.9"
 parking_lot = "0.12"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index d8ad36d1..e1923ec6 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage-bench"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine bench tool for foyer - the hybrid cache for Rust"
@@ -17,10 +17,10 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
-foyer-common = { version = "0.2", path = "../foyer-common" }
-foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.3", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.3", path = "../foyer-common" }
+foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.4", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
 http-body-util = "0.1"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index fb5ddd11..1b370d80 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
@@ -18,9 +18,9 @@ bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { version = "0.2", path = "../foyer-common" }
-foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
-foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.3", path = "../foyer-common" }
+foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.12"
 libc = "0.2"
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index a1c46186..a7683c7d 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -4,7 +4,7 @@
 
 [package]
 name = "foyer-workspace-hack"
-version = "0.1.1"
+version = "0.2.0"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "workspace-hack package, managed by hakari"
 license = "Apache-2.0"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 4e710255..dadb9798 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -13,7 +13,7 @@ homepage = "https://github.com/mrcroxx/foyer"
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
-foyer-common = { version = "0.2", path = "../foyer-common" }
-foyer-intrusive = { version = "0.1", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.3", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.1", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.3", path = "../foyer-common" }
+foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.4", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }

From e65b0a5bd626a56e2e68eaaae939d4920f1eccfc Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 19 Jan 2024 16:55:33 +0800
Subject: [PATCH 206/261] chore: update license (#258)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/batch.rs                     |   2 +-
 foyer-common/src/bits.rs                      |   2 +-
 foyer-common/src/code.rs                      |   2 +-
 foyer-common/src/continuum.rs                 |   2 +-
 foyer-common/src/erwlock.rs                   |   2 +-
 foyer-common/src/lib.rs                       |   2 +-
 foyer-common/src/queue.rs                     |   2 +-
 foyer-common/src/range.rs                     |   2 +-
 foyer-common/src/rate.rs                      |   2 +-
 foyer-common/src/rated_ticket.rs              |   2 +-
 foyer-common/src/runtime.rs                   |   2 +-
 .../examples/tombstone-log-bench.rs           | 111 ++++++
 foyer-experimental/src/error.rs               |  65 ++++
 foyer-experimental/src/tombstone.rs           | 329 ++++++++++++++++++
 foyer-intrusive/src/collections/dlist.rs      |   2 +-
 .../src/collections/duplicated_hashmap.rs     |   2 +-
 foyer-intrusive/src/collections/hashmap.rs    |   2 +-
 foyer-intrusive/src/collections/mod.rs        |   2 +-
 foyer-intrusive/src/core/adapter.rs           |   2 +-
 foyer-intrusive/src/core/mod.rs               |   2 +-
 foyer-intrusive/src/core/pointer.rs           |   2 +-
 foyer-intrusive/src/eviction/fifo.rs          |   2 +-
 foyer-intrusive/src/eviction/lfu.rs           |   2 +-
 foyer-intrusive/src/eviction/lru.rs           |   2 +-
 foyer-intrusive/src/eviction/mod.rs           |   2 +-
 foyer-intrusive/src/eviction/sfifo.rs         |   2 +-
 foyer-intrusive/src/lib.rs                    |   2 +-
 foyer-memory/src/lib.rs                       |   2 +-
 foyer-storage-bench/src/analyze.rs            |   2 +-
 foyer-storage-bench/src/export.rs             |   2 +-
 foyer-storage-bench/src/main.rs               |   2 +-
 foyer-storage-bench/src/rate.rs               |   2 +-
 foyer-storage-bench/src/text.rs               |   2 +-
 foyer-storage-bench/src/utils.rs              |   2 +-
 foyer-storage/src/admission/mod.rs            |   2 +-
 foyer-storage/src/admission/rated_ticket.rs   |   2 +-
 foyer-storage/src/buffer.rs                   |   2 +-
 foyer-storage/src/catalog.rs                  |   2 +-
 foyer-storage/src/compress.rs                 |   2 +-
 foyer-storage/src/device/allocator.rs         |   2 +-
 foyer-storage/src/device/error.rs             |   2 +-
 foyer-storage/src/device/fs.rs                |   2 +-
 foyer-storage/src/device/mod.rs               |   2 +-
 foyer-storage/src/error.rs                    |   2 +-
 foyer-storage/src/flusher.rs                  |   2 +-
 foyer-storage/src/generic.rs                  |   2 +-
 foyer-storage/src/indices.rs                  |   2 +-
 foyer-storage/src/judge.rs                    |   2 +-
 foyer-storage/src/lazy.rs                     |   2 +-
 foyer-storage/src/lib.rs                      |   2 +-
 foyer-storage/src/metrics.rs                  |   2 +-
 foyer-storage/src/reclaimer.rs                |   2 +-
 foyer-storage/src/region.rs                   |   2 +-
 foyer-storage/src/region_manager.rs           |   2 +-
 foyer-storage/src/reinsertion/exist.rs        |   2 +-
 foyer-storage/src/reinsertion/mod.rs          |   2 +-
 foyer-storage/src/reinsertion/rated_ticket.rs |   2 +-
 foyer-storage/src/runtime.rs                  |   2 +-
 foyer-storage/src/storage.rs                  |   2 +-
 foyer-storage/src/store.rs                    |   2 +-
 foyer-storage/src/test_utils.rs               |   2 +-
 foyer-storage/tests/storage_test.rs           |   2 +-
 foyer-workspace-hack/build.rs                 |   2 +-
 foyer-workspace-hack/src/lib.rs               |   2 +-
 foyer/src/lib.rs                              |   2 +-
 65 files changed, 567 insertions(+), 62 deletions(-)
 create mode 100644 foyer-experimental/examples/tombstone-log-bench.rs
 create mode 100644 foyer-experimental/src/error.rs
 create mode 100644 foyer-experimental/src/tombstone.rs

diff --git a/foyer-common/src/batch.rs b/foyer-common/src/batch.rs
index 9d2d6b79..b4300a49 100644
--- a/foyer-common/src/batch.rs
+++ b/foyer-common/src/batch.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/bits.rs b/foyer-common/src/bits.rs
index 7a6e24f6..378b3747 100644
--- a/foyer-common/src/bits.rs
+++ b/foyer-common/src/bits.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 4a363d23..6695fb21 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
index 430c33b9..a9fa147c 100644
--- a/foyer-common/src/continuum.rs
+++ b/foyer-common/src/continuum.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/erwlock.rs b/foyer-common/src/erwlock.rs
index 207734b7..2cee08b1 100644
--- a/foyer-common/src/erwlock.rs
+++ b/foyer-common/src/erwlock.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 899032c1..88d0173f 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/queue.rs b/foyer-common/src/queue.rs
index 03df30c0..c63e0eda 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/queue.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/range.rs b/foyer-common/src/range.rs
index 14679269..beef3900 100644
--- a/foyer-common/src/range.rs
+++ b/foyer-common/src/range.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/rate.rs b/foyer-common/src/rate.rs
index 1dcb0cb8..fdb55ce2 100644
--- a/foyer-common/src/rate.rs
+++ b/foyer-common/src/rate.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/rated_ticket.rs b/foyer-common/src/rated_ticket.rs
index 3cf94707..4673f2e0 100644
--- a/foyer-common/src/rated_ticket.rs
+++ b/foyer-common/src/rated_ticket.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index 8b002b84..e97581e1 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental/examples/tombstone-log-bench.rs b/foyer-experimental/examples/tombstone-log-bench.rs
new file mode 100644
index 00000000..6bc0f1f4
--- /dev/null
+++ b/foyer-experimental/examples/tombstone-log-bench.rs
@@ -0,0 +1,111 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    sync::Arc,
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
+};
+
+use clap::Parser;
+use foyer_common::runtime::BackgroundShutdownRuntime;
+use foyer_experimental::tombstone::{Tombstone, TombstoneLog, TombstoneLogConfig};
+use itertools::Itertools;
+use rand::{rngs::StdRng, Rng, SeedableRng};
+
+#[derive(Parser, Debug, Clone)]
+#[command(author, version, about)]
+pub struct Args {
+    /// dir for cache data
+    #[arg(short, long)]
+    dir: String,
+
+    /// writer concurrency
+    #[arg(short, long, default_value_t = 1024)]
+    concurrency: usize,
+
+    /// time (s)
+    #[arg(short, long, default_value_t = 10)]
+    time: usize,
+}
+
+#[tokio::main]
+async fn main() {
+    let args = Args::parse();
+
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+    let rt = BackgroundShutdownRuntime::from(rt);
+    let rt = Arc::new(rt);
+
+    let config = TombstoneLogConfig {
+        id: 0,
+        dir: args.dir.clone().into(),
+    };
+    let log = TombstoneLog::open(config).await.unwrap();
+
+    let handles = (0..args.concurrency)
+        .map(|_| {
+            let log = log.clone();
+            let args = args.clone();
+            let rt = rt.clone();
+            tokio::spawn(async move {
+                write(log.clone(), args, rt).await;
+            })
+        })
+        .collect_vec();
+
+    for handle in handles {
+        handle.await.unwrap();
+    }
+
+    println!("Bench finishes.");
+
+    log.close().await.unwrap();
+}
+
+async fn write(log: TombstoneLog<u64>, args: Args, rt: Arc<BackgroundShutdownRuntime>) {
+    let start = Instant::now();
+    let mut log = log;
+
+    let mut rng = StdRng::seed_from_u64(
+        SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_nanos() as _,
+    );
+    loop {
+        if start.elapsed() >= Duration::from_secs(args.time as _) {
+            return;
+        }
+
+        // let now = Instant::now();
+
+        let tombstone = Tombstone::new(rng.gen(), rng.gen());
+        log = rt
+            .spawn(async move {
+                log.append(tombstone).await.unwrap();
+                log
+            })
+            .await
+            .unwrap();
+
+        // let duration = now.elapsed();
+
+        // if duration.as_micros() >= 1000 {
+        //     println!("slow: {:?}", duration);
+        // }
+    }
+}
diff --git a/foyer-experimental/src/error.rs b/foyer-experimental/src/error.rs
new file mode 100644
index 00000000..61500387
--- /dev/null
+++ b/foyer-experimental/src/error.rs
@@ -0,0 +1,65 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::backtrace::Backtrace;
+
+#[derive(thiserror::Error, Debug)]
+#[error("{0}")]
+pub struct Error(Box<ErrorInner>);
+
+#[derive(thiserror::Error, Debug)]
+#[error("{source:?}")]
+struct ErrorInner {
+    #[from]
+    source: ErrorKind,
+    backtrace: Backtrace,
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum ErrorKind {
+    #[error("io error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("other error: {0}")]
+    Other(#[from] anyhow::Error),
+}
+
+impl From<ErrorKind> for Error {
+    fn from(value: ErrorKind) -> Self {
+        value.into()
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(value: std::io::Error) -> Self {
+        value.into()
+    }
+}
+
+impl From<anyhow::Error> for Error {
+    fn from(value: anyhow::Error) -> Self {
+        value.into()
+    }
+}
+
+pub type Result<T> = core::result::Result<T, Error>;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_error_size() {
+        assert_eq!(std::mem::size_of::<Error>(), std::mem::size_of::<usize>());
+    }
+}
diff --git a/foyer-experimental/src/tombstone.rs b/foyer-experimental/src/tombstone.rs
new file mode 100644
index 00000000..d9f85661
--- /dev/null
+++ b/foyer-experimental/src/tombstone.rs
@@ -0,0 +1,329 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fs::{File, OpenOptions},
+    io::Write,
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::JoinHandle,
+    time::Instant,
+};
+
+use bytes::{Buf, BufMut};
+use crossbeam::channel;
+use parking_lot::{Condvar, Mutex};
+use tokio::sync::oneshot;
+
+use crate::{
+    asyncify,
+    buf::{BufExt, BufMutExt},
+    error::Result,
+};
+
+pub trait HashValue: Send + Sync + 'static + Eq + std::fmt::Debug {
+    fn size() -> usize;
+    fn write(&self, buf: impl BufMut);
+    fn read(buf: impl Buf) -> Self;
+}
+
+macro_rules! for_all_primitives {
+    ($macro:ident) => {
+        $macro! {
+            u8, u16, u32, u64, usize,
+            i8, i16, i32, i64, isize,
+        }
+    };
+}
+
+macro_rules! impl_hash_value {
+    ($( $type:ty, )*) => {
+        paste::paste! {
+            $(
+                impl HashValue for $type {
+                    fn size() -> usize {
+                        std::mem::size_of::<$type>()
+                    }
+
+                    fn write(&self, mut buf: impl BufMut) {
+                        buf.[<put_ $type>](*self);
+                    }
+
+                    fn read(mut buf: impl Buf) -> Self {
+                        buf.[<get_ $type>]()
+                    }
+                }
+            )*
+        }
+    };
+}
+
+for_all_primitives! { impl_hash_value }
+
+#[derive(Debug)]
+pub struct Tombstone<H: HashValue> {
+    hash: H,
+    pos: u32,
+}
+
+impl<H: HashValue> Tombstone<H> {
+    pub fn size() -> usize {
+        H::size() + 8
+    }
+
+    pub fn new(hash: H, pos: u32) -> Self {
+        Self { hash, pos }
+    }
+
+    pub fn write(&self, mut buf: impl BufMut) {
+        self.hash.write(&mut buf);
+        buf.put_u32(self.pos);
+    }
+
+    pub fn read(mut buf: impl Buf) -> Self {
+        let hash = H::read(&mut buf);
+        let pos = buf.get_u32();
+        Self { hash, pos }
+    }
+}
+
+#[derive(Debug)]
+pub struct TombstoneLogConfig {
+    pub id: usize,
+    pub dir: PathBuf,
+}
+
+#[derive(Debug)]
+struct InflightTombstone<H: HashValue> {
+    tombstone: Tombstone<H>,
+    tx: oneshot::Sender<Result<()>>,
+}
+
+#[derive(Debug)]
+struct TombstoneLogInner<H: HashValue> {
+    inflights: Mutex<Vec<InflightTombstone<H>>>,
+    condvar: Condvar,
+}
+
+#[derive(Debug)]
+pub struct TombstoneLog<H: HashValue> {
+    inner: Arc<TombstoneLogInner<H>>,
+    stopped: Arc<AtomicBool>,
+    handles: Arc<Mutex<Vec<JoinHandle<Result<()>>>>>,
+}
+
+impl<H: HashValue> Clone for TombstoneLog<H> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+            stopped: self.stopped.clone(),
+            handles: self.handles.clone(),
+        }
+    }
+}
+
+impl<H: HashValue> TombstoneLog<H> {
+    pub async fn open(config: TombstoneLogConfig) -> Result<Self> {
+        let mut path = config.dir;
+
+        path.push(format!("tombstone-{:08X}", config.id));
+
+        let file = OpenOptions::new()
+            .write(true)
+            .read(true)
+            .create(true)
+            .open(path)?;
+
+        let inner = Arc::new(TombstoneLogInner {
+            inflights: Mutex::new(vec![]),
+            condvar: Condvar::new(),
+        });
+
+        let (task_tx, task_rx) = channel::unbounded();
+        let stopped = Arc::new(AtomicBool::new(false));
+
+        let flusher = TombstoneLogFlusher {
+            file,
+            inner: inner.clone(),
+            task_tx,
+            stopped: stopped.clone(),
+        };
+
+        let notifier = TombstoneLogFlushNotifier { task_rx };
+
+        let handles = vec![
+            std::thread::spawn(move || flusher.run()),
+            std::thread::spawn(move || notifier.run()),
+        ];
+        let handles = Arc::new(Mutex::new(handles));
+
+        Ok(Self {
+            inner,
+            stopped,
+            handles,
+        })
+    }
+
+    pub async fn close(&self) -> Result<()> {
+        let handles = std::mem::take(&mut *self.handles.lock());
+        if !handles.is_empty() {
+            self.stopped.store(true, Ordering::Release);
+            self.inner.condvar.notify_one();
+            for handle in handles {
+                asyncify(move || handle.join().unwrap()).await?;
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn append(&self, tombstone: Tombstone<H>) -> Result<()> {
+        let rx = {
+            let mut inflights = self.inner.inflights.lock();
+            let (tx, rx) = oneshot::channel();
+            inflights.push(InflightTombstone { tombstone, tx });
+            rx
+        };
+        self.inner.condvar.notify_one();
+        rx.await.unwrap()
+    }
+}
+
+#[derive(Debug)]
+struct TombstoneLogFlusher<H: HashValue> {
+    file: File,
+    inner: Arc<TombstoneLogInner<H>>,
+    task_tx: channel::Sender<FlushNotifyTask>,
+    stopped: Arc<AtomicBool>,
+}
+
+impl<H: HashValue> TombstoneLogFlusher<H> {
+    fn run(mut self) -> Result<()> {
+        loop {
+            let inflights = {
+                let mut inflights = self.inner.inflights.lock();
+                if self.stopped.load(Ordering::Relaxed) {
+                    return Ok(());
+                }
+                if inflights.is_empty() {
+                    self.inner.condvar.wait_while(&mut inflights, |inflights| {
+                        inflights.is_empty() && !self.stopped.load(Ordering::Relaxed)
+                    });
+                }
+                if self.stopped.load(Ordering::Relaxed) {
+                    return Ok(());
+                }
+                std::mem::take(&mut *inflights)
+            };
+
+            let now = Instant::now();
+
+            let mut buffer = Vec::with_capacity(Tombstone::<H>::size() * inflights.len());
+            let mut txs = Vec::with_capacity(inflights.len());
+            let ts = inflights.len();
+
+            let write_buffer_start = Instant::now();
+            for inflight in inflights {
+                inflight.tombstone.write(&mut buffer);
+                txs.push(inflight.tx);
+            }
+            let write_buffer_duration = write_buffer_start.elapsed();
+
+            let write_all_start = Instant::now();
+            match self.file.write_all(&buffer) {
+                Ok(()) => {}
+                Err(e) => {
+                    self.task_tx
+                        .send(FlushNotifyTask {
+                            txs,
+                            io_result: Err(e),
+                        })
+                        .unwrap();
+                    continue;
+                }
+            }
+            let write_all_duration = write_all_start.elapsed();
+
+            let fdatasync_start = Instant::now();
+            match self.file.sync_data() {
+                Ok(()) => {}
+                Err(e) => {
+                    self.task_tx
+                        .send(FlushNotifyTask {
+                            txs,
+                            io_result: Err(e),
+                        })
+                        .unwrap();
+                    continue;
+                }
+            }
+            let fdatasync_duration = fdatasync_start.elapsed();
+
+            let dispatch_start = Instant::now();
+            self.task_tx
+                .send(FlushNotifyTask {
+                    txs,
+                    io_result: Ok(()),
+                })
+                .unwrap();
+            let dispatch_duration = dispatch_start.elapsed();
+
+            let duration = now.elapsed();
+
+            if duration.as_micros() >= 500 {
+                println!(
+                    "slow: {:?}, write buffer: {:?}, write all: {:?}, fdatasync: {:?}, dispatch: {:?}, ts: {:?}, buffer size: {:.3}KB",
+                    duration,
+                    write_buffer_duration,
+                    write_all_duration,
+                    fdatasync_duration,
+                    dispatch_duration,
+                    ts,
+                    buffer.len() as f64 / 4096.0,
+                );
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+struct FlushNotifyTask {
+    txs: Vec<oneshot::Sender<Result<()>>>,
+    io_result: std::io::Result<()>,
+}
+
+#[derive(Debug)]
+struct TombstoneLogFlushNotifier {
+    task_rx: channel::Receiver<FlushNotifyTask>,
+}
+
+impl TombstoneLogFlushNotifier {
+    fn run(self) -> Result<()> {
+        while let Ok(task) = self.task_rx.recv() {
+            for tx in task.txs {
+                let res = match &task.io_result {
+                    Ok(()) => Ok(()),
+                    Err(e) => Err(std::io::Error::from(e.kind()).into()),
+                };
+                tx.send(res).unwrap();
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {}
diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index 38c35f5c..ddd8c8bc 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index 13075297..49c2494b 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index c1bd07f2..961d0566 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/collections/mod.rs b/foyer-intrusive/src/collections/mod.rs
index 265cf55d..d3b15ac6 100644
--- a/foyer-intrusive/src/collections/mod.rs
+++ b/foyer-intrusive/src/collections/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index e245506f..32057323 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/core/mod.rs b/foyer-intrusive/src/core/mod.rs
index 6c823b94..f79f4215 100644
--- a/foyer-intrusive/src/core/mod.rs
+++ b/foyer-intrusive/src/core/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/core/pointer.rs b/foyer-intrusive/src/core/pointer.rs
index ef0cfc74..53c6dcc8 100644
--- a/foyer-intrusive/src/core/pointer.rs
+++ b/foyer-intrusive/src/core/pointer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index dc4f60ba..494b9b4e 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index ed478a65..6893fab2 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index e287c9c6..2915b0e5 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index aebc72f7..4338532c 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index cd422db0..a7dca2ad 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index a8126dac..19104d42 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 93aabd63..e695f93e 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index e518403b..bf9190dc 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/export.rs b/foyer-storage-bench/src/export.rs
index 3fcf7d10..2d918221 100644
--- a/foyer-storage-bench/src/export.rs
+++ b/foyer-storage-bench/src/export.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 6c5f5566..1a927ad5 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/rate.rs b/foyer-storage-bench/src/rate.rs
index 82164aab..5be56a63 100644
--- a/foyer-storage-bench/src/rate.rs
+++ b/foyer-storage-bench/src/rate.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/text.rs b/foyer-storage-bench/src/text.rs
index dceae35d..614d4365 100644
--- a/foyer-storage-bench/src/text.rs
+++ b/foyer-storage-bench/src/text.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
index 5c123a5d..cc486d32 100644
--- a/foyer-storage-bench/src/utils.rs
+++ b/foyer-storage-bench/src/utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index e966d6b4..670f27b2 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
index c886571e..2a9c8884 100644
--- a/foyer-storage/src/admission/rated_ticket.rs
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 48d88b3e..cbf1c7d3 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 358791ee..0cf6bf7d 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/compress.rs b/foyer-storage/src/compress.rs
index 53cada0f..20020ca5 100644
--- a/foyer-storage/src/compress.rs
+++ b/foyer-storage/src/compress.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/allocator.rs b/foyer-storage/src/device/allocator.rs
index cb133602..94902deb 100644
--- a/foyer-storage/src/device/allocator.rs
+++ b/foyer-storage/src/device/allocator.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/error.rs b/foyer-storage/src/device/error.rs
index 5b595cda..01e9e135 100644
--- a/foyer-storage/src/device/error.rs
+++ b/foyer-storage/src/device/error.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index dbafab04..22cc7752 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index c5f9e39f..c363ee72 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index e18363a0..ea7d055c 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 0ad99c61..8bb2d8fb 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 094dbcfa..41c2cec1 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/indices.rs b/foyer-storage/src/indices.rs
index 800da36d..1cd2cf5b 100644
--- a/foyer-storage/src/indices.rs
+++ b/foyer-storage/src/indices.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/judge.rs b/foyer-storage/src/judge.rs
index c3fa7bdd..3b1e121d 100644
--- a/foyer-storage/src/judge.rs
+++ b/foyer-storage/src/judge.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 1b4b6e3e..19e85c9e 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 94155679..1578f5d8 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 9a67a6f8..ad9b3d94 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index d35f4830..ba4c4002 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 5f519df0..45d33481 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 81167307..2578d3e8 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index 2dda6a31..2e10d50f 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index 3fe93f93..e39b08a0 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
index 46c06e74..2e313faa 100644
--- a/foyer-storage/src/reinsertion/rated_ticket.rs
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 2dd931cf..1f9b7e40 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 074306be..03488f8f 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 538cfc5f..a8058ed9 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index be7fe4cf..1ca8541a 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 46459f32..48f01bbc 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-workspace-hack/build.rs b/foyer-workspace-hack/build.rs
index b7889cc7..540913fa 100644
--- a/foyer-workspace-hack/build.rs
+++ b/foyer-workspace-hack/build.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-workspace-hack/src/lib.rs b/foyer-workspace-hack/src/lib.rs
index 61f71adc..fffa08f9 100644
--- a/foyer-workspace-hack/src/lib.rs
+++ b/foyer-workspace-hack/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 0ea25c6d..e15cb512 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2023 MrCroxx
+//  Copyright 2024 MrCroxx
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.

From 09b8b104cbe970bf2ba4eb47ddb15521ffbd9345 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 19 Jan 2024 17:28:59 +0800
Subject: [PATCH 207/261] feat(experimental): introduce tombstone log (#257)

* feat(experimental): introduce tombstone log

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .typos.toml                     |  8 ++++
 Cargo.toml                      |  2 +
 foyer-experimental/Cargo.toml   | 41 ++++++++++++++++
 foyer-experimental/src/buf.rs   | 83 +++++++++++++++++++++++++++++++++
 foyer-experimental/src/lib.rs   | 41 ++++++++++++++++
 foyer-workspace-hack/Cargo.toml |  3 +-
 6 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 .typos.toml
 create mode 100644 foyer-experimental/Cargo.toml
 create mode 100644 foyer-experimental/src/buf.rs
 create mode 100644 foyer-experimental/src/lib.rs

diff --git a/.typos.toml b/.typos.toml
new file mode 100644
index 00000000..d843d98a
--- /dev/null
+++ b/.typos.toml
@@ -0,0 +1,8 @@
+[default.extend-words]
+
+[default.extend-identifiers]
+
+[files]
+extend-exclude = [
+    "foyer-workspace-hack/Cargo.toml",
+]
diff --git a/Cargo.toml b/Cargo.toml
index f5245f18..554cb2b4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,6 +3,7 @@ resolver = "2"
 members = [
     "foyer",
     "foyer-common",
+    "foyer-experimental",
     "foyer-intrusive",
     "foyer-memory",
     "foyer-storage",
@@ -18,6 +19,7 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
     "macros",
     "time",
     "signal",
+    "fs",
 ] }
 
 [patch.crates-io]
diff --git a/foyer-experimental/Cargo.toml b/foyer-experimental/Cargo.toml
new file mode 100644
index 00000000..aea40e21
--- /dev/null
+++ b/foyer-experimental/Cargo.toml
@@ -0,0 +1,41 @@
+[package]
+name = "foyer-experimental"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "experimental components for foyer - the hybrid cache for Rust"
+license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
+
+[dependencies]
+anyhow = "1.0"
+bytes = "1"
+crossbeam = { version = "0.8", features = ["crossbeam-channel"] }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+parking_lot = { version = "0.12", features = ["arc_lock"] }
+paste = "1.0"
+thiserror = "1"
+tokio = { workspace = true }
+tracing = "0.1"
+
+[dev-dependencies]
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+foyer-common = { version = "0.3", path = "../foyer-common" }
+hdrhistogram = "7"
+itertools = "0.12"
+rand = "0.8.5"
+rand_mt = "4.2.1"
+tempfile = "3"
+
+[features]
+deadlock = ["parking_lot/deadlock_detection"]
+
+[[example]]
+name = "tombstone-log-bench"
+
diff --git a/foyer-experimental/src/buf.rs b/foyer-experimental/src/buf.rs
new file mode 100644
index 00000000..28270cc8
--- /dev/null
+++ b/foyer-experimental/src/buf.rs
@@ -0,0 +1,83 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use bytes::{Buf, BufMut};
+
+pub trait BufExt: Buf {
+    cfg_match! {
+        cfg(target_pointer_width = "16") => {
+            fn get_usize(&mut self) -> usize {
+                self.get_u16() as usize
+            }
+
+            fn get_isize(&mut self) -> isize {
+                self.get_i16() as isize
+            }
+        }
+        cfg(target_pointer_width = "32") => {
+            fn get_usize(&mut self) -> usize {
+                self.get_u32() as usize
+            }
+
+            fn get_isize(&mut self) -> isize {
+                self.get_i32() as isize
+            }
+        }
+        cfg(target_pointer_width = "64") => {
+            fn get_usize(&mut self) -> usize {
+                self.get_u64() as usize
+            }
+
+            fn get_isize(&mut self) -> isize {
+                self.get_i64() as isize
+            }
+        }
+    }
+}
+
+impl<T: Buf> BufExt for T {}
+
+pub trait BufMutExt: BufMut {
+    cfg_match! {
+        cfg(target_pointer_width = "16") => {
+            fn put_usize(&mut self, v: usize) {
+                self.put_u16(v as u16);
+            }
+
+            fn put_isize(&mut self, v: isize) {
+                self.put_i16(v as i16);
+            }
+        }
+        cfg(target_pointer_width = "32") => {
+            fn put_usize(&mut self, v: usize) {
+                self.put_u32(v as u32);
+            }
+
+            fn put_isize(&mut self, v: isize) {
+                self.put_i32(v as i32);
+            }
+        }
+        cfg(target_pointer_width = "64") => {
+            fn put_usize(&mut self, v: usize) {
+                self.put_u64(v as u64);
+            }
+
+            fn put_isize(&mut self, v: isize) {
+                self.put_i64(v as i64);
+            }
+        }
+    }
+}
+
+impl<T: BufMut> BufMutExt for T {}
diff --git a/foyer-experimental/src/lib.rs b/foyer-experimental/src/lib.rs
new file mode 100644
index 00000000..f8d975ef
--- /dev/null
+++ b/foyer-experimental/src/lib.rs
@@ -0,0 +1,41 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![feature(cfg_match)]
+#![feature(lint_reasons)]
+#![feature(error_generic_member_access)]
+
+pub mod buf;
+pub mod error;
+pub mod tombstone;
+
+#[cfg(not(madsim))]
+#[tracing::instrument(level = "trace", skip(f))]
+async fn asyncify<F, T>(f: F) -> T
+where
+    F: FnOnce() -> T + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::task::spawn_blocking(f).await.unwrap()
+}
+
+#[cfg(madsim)]
+#[tracing::instrument(level = "trace", skip(f))]
+async fn asyncify<F, T>(f: F) -> T
+where
+    F: FnOnce() -> T + Send + 'static,
+    T: Send + 'static,
+{
+    f()
+}
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index a7683c7d..1d436b89 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -17,6 +17,7 @@ publish = true
 
 ### BEGIN HAKARI SECTION
 [dependencies]
+crossbeam-channel = { version = "0.5" }
 crossbeam-utils = { version = "0.8" }
 either = { version = "1", default-features = false, features = ["use_std"] }
 futures-channel = { version = "0.3", features = ["sink"] }
@@ -29,7 +30,7 @@ memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
-tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
+tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 tracing-core = { version = "0.1" }
 
 [build-dependencies]

From e837fa95576bb7110e2c680fad4b08e94b080453 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 23 Jan 2024 12:59:38 +0800
Subject: [PATCH 208/261] refactor: refine foyer-experimental-bench structure
 (#259)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml                                    |   1 +
 codecov.yml                                   |   1 +
 foyer-experimental-bench/Cargo.toml           |  67 ++
 .../benches/wal-bench.rs                      |  41 +-
 foyer-experimental-bench/etc/sample.txt       | 676 ++++++++++++++++++
 foyer-experimental-bench/src/analyze.rs       | 353 +++++++++
 foyer-experimental-bench/src/export.rs        |  70 ++
 foyer-experimental-bench/src/lib.rs           | 193 +++++
 foyer-experimental-bench/src/rate.rs          |  59 ++
 foyer-experimental-bench/src/text.rs          |  28 +
 foyer-experimental-bench/src/utils.rs         | 170 +++++
 foyer-experimental/Cargo.toml                 |   5 +-
 foyer-experimental/src/lib.rs                 |   4 +-
 foyer-experimental/src/metrics.rs             | 204 ++++++
 .../src/{tombstone.rs => wal.rs}              |  66 +-
 foyer-storage/src/reclaimer.rs                |   1 -
 16 files changed, 1894 insertions(+), 45 deletions(-)
 create mode 100644 foyer-experimental-bench/Cargo.toml
 rename foyer-experimental/examples/tombstone-log-bench.rs => foyer-experimental-bench/benches/wal-bench.rs (76%)
 create mode 100644 foyer-experimental-bench/etc/sample.txt
 create mode 100644 foyer-experimental-bench/src/analyze.rs
 create mode 100644 foyer-experimental-bench/src/export.rs
 create mode 100644 foyer-experimental-bench/src/lib.rs
 create mode 100644 foyer-experimental-bench/src/rate.rs
 create mode 100644 foyer-experimental-bench/src/text.rs
 create mode 100644 foyer-experimental-bench/src/utils.rs
 create mode 100644 foyer-experimental/src/metrics.rs
 rename foyer-experimental/src/{tombstone.rs => wal.rs} (86%)

diff --git a/Cargo.toml b/Cargo.toml
index 554cb2b4..ba63fa62 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,7 @@ members = [
     "foyer",
     "foyer-common",
     "foyer-experimental",
+    "foyer-experimental-bench",
     "foyer-intrusive",
     "foyer-memory",
     "foyer-storage",
diff --git a/codecov.yml b/codecov.yml
index 873b19f0..0907d437 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -22,3 +22,4 @@ coverage:
         only_pulls: true
 ignore:
   - "foyer-storage-bench"
+  - "foyer-experimental-bench"
diff --git a/foyer-experimental-bench/Cargo.toml b/foyer-experimental-bench/Cargo.toml
new file mode 100644
index 00000000..2c507afc
--- /dev/null
+++ b/foyer-experimental-bench/Cargo.toml
@@ -0,0 +1,67 @@
+[package]
+name = "foyer-experimental-bench"
+version = "0.1.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "storage engine bench tool for foyer - the hybrid cache for Rust"
+license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["foyer-workspace-hack"]
+
+[dependencies]
+anyhow = "1"
+bytesize = "1"
+clap = { version = "4", features = ["derive"] }
+console-subscriber = { version = "0.2", optional = true }
+foyer-common = { version = "0.3", path = "../foyer-common" }
+foyer-experimental = { version = "0.1", path = "../foyer-experimental" }
+foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.4", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+futures = "0.3"
+hdrhistogram = "7"
+http-body-util = "0.1"
+hyper = { version = "1.0", features = ["server", "http1", "http2"] }
+hyper-util = { version = "0.1", features = [
+    "server",
+    "server-auto",
+    "http1",
+    "http2",
+    "tokio",
+] }
+itertools = "0.12"
+libc = "0.2"
+nix = { version = "0.27", features = ["fs", "mman"] }
+opentelemetry = { version = "0.21", optional = true }
+opentelemetry-otlp = { version = "0.14.0", optional = true }
+opentelemetry-semantic-conventions = { version = "0.13", optional = true }
+opentelemetry_sdk = { version = "0.21", features = [
+    "rt-tokio",
+    "trace",
+], optional = true }
+parking_lot = "0.12"
+prometheus = "0.13"
+rand = "0.8.5"
+tokio = { workspace = true }
+tracing = "0.1"
+tracing-opentelemetry = { version = "0.22", optional = true }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+[features]
+deadlock = ["parking_lot/deadlock_detection", "foyer-storage/deadlock"]
+tokio-console = ["console-subscriber"]
+trace = [
+    "opentelemetry",
+    "opentelemetry_sdk",
+    "opentelemetry-otlp",
+    "tracing-opentelemetry",
+    "opentelemetry-semantic-conventions",
+]
+
+[[bin]]
+name = "wal-bench"
+path = "benches/wal-bench.rs"
diff --git a/foyer-experimental/examples/tombstone-log-bench.rs b/foyer-experimental-bench/benches/wal-bench.rs
similarity index 76%
rename from foyer-experimental/examples/tombstone-log-bench.rs
rename to foyer-experimental-bench/benches/wal-bench.rs
index 6bc0f1f4..bc9470ba 100644
--- a/foyer-experimental/examples/tombstone-log-bench.rs
+++ b/foyer-experimental-bench/benches/wal-bench.rs
@@ -13,13 +13,18 @@
 //  limitations under the License.
 
 use std::{
+    path::PathBuf,
     sync::Arc,
     time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 
 use clap::Parser;
 use foyer_common::runtime::BackgroundShutdownRuntime;
-use foyer_experimental::tombstone::{Tombstone, TombstoneLog, TombstoneLogConfig};
+use foyer_experimental::{
+    metrics::METRICS,
+    wal::{Tombstone, TombstoneLog, TombstoneLogConfig},
+};
+use foyer_experimental_bench::{export::MetricsExporter, init_logger, io_monitor, IoMonitorConfig};
 use itertools::Itertools;
 use rand::{rngs::StdRng, Rng, SeedableRng};
 
@@ -35,14 +40,35 @@ pub struct Args {
     concurrency: usize,
 
     /// time (s)
-    #[arg(short, long, default_value_t = 10)]
+    #[arg(short, long, default_value_t = 60)]
     time: usize,
+
+    /// (s)
+    #[arg(long, default_value_t = 2)]
+    report_interval: u64,
+
+    #[arg(long, default_value_t = false)]
+    metrics: bool,
 }
 
 #[tokio::main]
 async fn main() {
     let args = Args::parse();
 
+    println!("{:#?}", args);
+
+    init_logger();
+
+    if args.metrics {
+        MetricsExporter::init("0.0.0.0:19970".parse().unwrap());
+    }
+
+    let guard = io_monitor(IoMonitorConfig {
+        dir: PathBuf::from(&args.dir),
+        interval: Duration::from_secs(args.report_interval),
+        total_secs: args.time,
+    });
+
     let rt = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()
@@ -53,6 +79,7 @@ async fn main() {
     let config = TombstoneLogConfig {
         id: 0,
         dir: args.dir.clone().into(),
+        metrics: Arc::new(METRICS.metrics("wal-bench")),
     };
     let log = TombstoneLog::open(config).await.unwrap();
 
@@ -71,7 +98,7 @@ async fn main() {
         handle.await.unwrap();
     }
 
-    println!("Bench finishes.");
+    drop(guard);
 
     log.close().await.unwrap();
 }
@@ -91,8 +118,6 @@ async fn write(log: TombstoneLog<u64>, args: Args, rt: Arc<BackgroundShutdownRun
             return;
         }
 
-        // let now = Instant::now();
-
         let tombstone = Tombstone::new(rng.gen(), rng.gen());
         log = rt
             .spawn(async move {
@@ -101,11 +126,5 @@ async fn write(log: TombstoneLog<u64>, args: Args, rt: Arc<BackgroundShutdownRun
             })
             .await
             .unwrap();
-
-        // let duration = now.elapsed();
-
-        // if duration.as_micros() >= 1000 {
-        //     println!("slow: {:?}", duration);
-        // }
     }
 }
diff --git a/foyer-experimental-bench/etc/sample.txt b/foyer-experimental-bench/etc/sample.txt
new file mode 100644
index 00000000..ab729b88
--- /dev/null
+++ b/foyer-experimental-bench/etc/sample.txt
@@ -0,0 +1,676 @@
+!!! This is just a sample text, not a real license. !!!
+
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.
diff --git a/foyer-experimental-bench/src/analyze.rs b/foyer-experimental-bench/src/analyze.rs
new file mode 100644
index 00000000..bf9190dc
--- /dev/null
+++ b/foyer-experimental-bench/src/analyze.rs
@@ -0,0 +1,353 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    path::Path,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
+
+use bytesize::ByteSize;
+use hdrhistogram::Histogram;
+use parking_lot::RwLock;
+use tokio::sync::broadcast;
+
+use crate::utils::{iostat, IoStat};
+
+const SECTOR_SIZE: usize = 512;
+
+// latencies are measured by 'us'
+#[derive(Clone, Copy, Debug)]
+pub struct Analysis {
+    disk_read_iops: f64,
+    disk_read_throughput: f64,
+    disk_write_iops: f64,
+    disk_write_throughput: f64,
+
+    insert_iops: f64,
+    insert_throughput: f64,
+    insert_lat_p50: u64,
+    insert_lat_p90: u64,
+    insert_lat_p99: u64,
+    insert_lat_p999: u64,
+    insert_lat_p9999: u64,
+    insert_lat_p99999: u64,
+    insert_lat_pmax: u64,
+
+    get_iops: f64,
+    get_miss: f64,
+    get_throughput: f64,
+    get_miss_lat_p50: u64,
+    get_miss_lat_p90: u64,
+    get_miss_lat_p99: u64,
+    get_miss_lat_p999: u64,
+    get_miss_lat_p9999: u64,
+    get_miss_lat_p99999: u64,
+    get_miss_lat_pmax: u64,
+    get_hit_lat_p50: u64,
+    get_hit_lat_p90: u64,
+    get_hit_lat_p99: u64,
+    get_hit_lat_p999: u64,
+    get_hit_lat_p9999: u64,
+    get_hit_lat_p99999: u64,
+    get_hit_lat_pmax: u64,
+}
+
+#[derive(Default, Clone, Copy, Debug)]
+pub struct MetricsDump {
+    pub insert_ios: usize,
+    pub insert_bytes: usize,
+    pub insert_lat_p50: u64,
+    pub insert_lat_p90: u64,
+    pub insert_lat_p99: u64,
+    pub insert_lat_p999: u64,
+    pub insert_lat_p9999: u64,
+    pub insert_lat_p99999: u64,
+    pub insert_lat_pmax: u64,
+
+    pub get_ios: usize,
+    pub get_miss_ios: usize,
+    pub get_bytes: usize,
+    pub get_hit_lat_p50: u64,
+    pub get_hit_lat_p90: u64,
+    pub get_hit_lat_p99: u64,
+    pub get_hit_lat_p999: u64,
+    pub get_hit_lat_p9999: u64,
+    pub get_hit_lat_p99999: u64,
+    pub get_hit_lat_pmax: u64,
+    pub get_miss_lat_p50: u64,
+    pub get_miss_lat_p90: u64,
+    pub get_miss_lat_p99: u64,
+    pub get_miss_lat_p999: u64,
+    pub get_miss_lat_p9999: u64,
+    pub get_miss_lat_p99999: u64,
+    pub get_miss_lat_pmax: u64,
+}
+
+#[derive(Clone, Debug)]
+pub struct Metrics {
+    pub insert_ios: Arc<AtomicUsize>,
+    pub insert_bytes: Arc<AtomicUsize>,
+    pub insert_lats: Arc<RwLock<Histogram<u64>>>,
+
+    pub get_ios: Arc<AtomicUsize>,
+    pub get_bytes: Arc<AtomicUsize>,
+    pub get_miss_ios: Arc<AtomicUsize>,
+    pub get_hit_lats: Arc<RwLock<Histogram<u64>>>,
+    pub get_miss_lats: Arc<RwLock<Histogram<u64>>>,
+}
+
+impl Default for Metrics {
+    fn default() -> Self {
+        Self {
+            insert_ios: Arc::new(AtomicUsize::new(0)),
+            insert_bytes: Arc::new(AtomicUsize::new(0)),
+            insert_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+
+            get_ios: Arc::new(AtomicUsize::new(0)),
+            get_bytes: Arc::new(AtomicUsize::new(0)),
+            get_miss_ios: Arc::new(AtomicUsize::new(0)),
+            get_hit_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+            get_miss_lats: Arc::new(RwLock::new(
+                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
+            )),
+        }
+    }
+}
+
+impl Metrics {
+    pub fn dump(&self) -> MetricsDump {
+        let insert_lats = self.insert_lats.read();
+        let get_hit_lats = self.get_hit_lats.read();
+        let get_miss_lats = self.get_miss_lats.read();
+
+        MetricsDump {
+            insert_ios: self.insert_ios.load(Ordering::Relaxed),
+            insert_bytes: self.insert_bytes.load(Ordering::Relaxed),
+            insert_lat_p50: insert_lats.value_at_quantile(0.5),
+            insert_lat_p90: insert_lats.value_at_quantile(0.9),
+            insert_lat_p99: insert_lats.value_at_quantile(0.99),
+            insert_lat_p999: insert_lats.value_at_quantile(0.999),
+            insert_lat_p9999: insert_lats.value_at_quantile(0.9999),
+            insert_lat_p99999: insert_lats.value_at_quantile(0.99999),
+            insert_lat_pmax: insert_lats.max(),
+
+            get_ios: self.get_ios.load(Ordering::Relaxed),
+            get_miss_ios: self.get_miss_ios.load(Ordering::Relaxed),
+            get_bytes: self.get_bytes.load(Ordering::Relaxed),
+            get_hit_lat_p50: get_hit_lats.value_at_quantile(0.5),
+            get_hit_lat_p90: get_hit_lats.value_at_quantile(0.9),
+            get_hit_lat_p99: get_hit_lats.value_at_quantile(0.99),
+            get_hit_lat_p999: get_hit_lats.value_at_quantile(0.999),
+            get_hit_lat_p9999: get_hit_lats.value_at_quantile(0.9999),
+            get_hit_lat_p99999: get_hit_lats.value_at_quantile(0.99999),
+            get_hit_lat_pmax: get_hit_lats.max(),
+            get_miss_lat_p50: get_miss_lats.value_at_quantile(0.5),
+            get_miss_lat_p90: get_miss_lats.value_at_quantile(0.9),
+            get_miss_lat_p99: get_miss_lats.value_at_quantile(0.99),
+            get_miss_lat_p999: get_miss_lats.value_at_quantile(0.999),
+            get_miss_lat_p9999: get_miss_lats.value_at_quantile(0.9999),
+            get_miss_lat_p99999: get_miss_lats.value_at_quantile(0.99999),
+            get_miss_lat_pmax: get_miss_lats.max(),
+        }
+    }
+}
+
+impl std::fmt::Display for Analysis {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let disk_read_throughput = ByteSize::b(self.disk_read_throughput as u64);
+        let disk_write_throughput = ByteSize::b(self.disk_write_throughput as u64);
+        let disk_total_throughput = disk_read_throughput + disk_write_throughput;
+
+        // disk statics
+        writeln!(
+            f,
+            "disk total iops: {:.1}",
+            self.disk_write_iops + self.disk_read_iops
+        )?;
+        writeln!(
+            f,
+            "disk total throughput: {}/s",
+            disk_total_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "disk read iops: {:.1}", self.disk_read_iops)?;
+        writeln!(
+            f,
+            "disk read throughput: {}/s",
+            disk_read_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "disk write iops: {:.1}", self.disk_write_iops)?;
+        writeln!(
+            f,
+            "disk write throughput: {}/s",
+            disk_write_throughput.to_string_as(true)
+        )?;
+
+        // insert statics
+        let insert_throughput = ByteSize::b(self.insert_throughput as u64);
+        writeln!(f, "insert iops: {:.1}/s", self.insert_iops)?;
+        writeln!(
+            f,
+            "insert throughput: {}/s",
+            insert_throughput.to_string_as(true)
+        )?;
+        writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
+        writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
+        writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
+        writeln!(f, "insert lat p999: {}us", self.insert_lat_p999)?;
+        writeln!(f, "insert lat p9999: {}us", self.insert_lat_p9999)?;
+        writeln!(f, "insert lat p99999: {}us", self.insert_lat_p99999)?;
+        writeln!(f, "insert lat pmax: {}us", self.insert_lat_pmax)?;
+
+        // get statics
+        let get_throughput = ByteSize::b(self.get_throughput as u64);
+        writeln!(f, "get iops: {:.1}/s", self.get_iops)?;
+        writeln!(f, "get miss: {:.2}% ", self.get_miss * 100f64)?;
+        writeln!(f, "get throughput: {}/s", get_throughput.to_string_as(true))?;
+        writeln!(f, "get hit lat p50: {}us", self.get_hit_lat_p50)?;
+        writeln!(f, "get hit lat p90: {}us", self.get_hit_lat_p90)?;
+        writeln!(f, "get hit lat p99: {}us", self.get_hit_lat_p99)?;
+        writeln!(f, "get hit lat p999: {}us", self.get_hit_lat_p999)?;
+        writeln!(f, "get hit lat p9999: {}us", self.get_hit_lat_p9999)?;
+        writeln!(f, "get hit lat p99999: {}us", self.get_hit_lat_p99999)?;
+        writeln!(f, "get hit lat pmax: {}us", self.get_hit_lat_pmax)?;
+        writeln!(f, "get miss lat p50: {}us", self.get_miss_lat_p50)?;
+        writeln!(f, "get miss lat p90: {}us", self.get_miss_lat_p90)?;
+        writeln!(f, "get miss lat p99: {}us", self.get_miss_lat_p99)?;
+        writeln!(f, "get miss lat p999: {}us", self.get_miss_lat_p999)?;
+        writeln!(f, "get miss lat p9999: {}us", self.get_miss_lat_p9999)?;
+        writeln!(f, "get miss lat p99999: {}us", self.get_miss_lat_p99999)?;
+        writeln!(f, "get miss lat pmax: {}us", self.get_miss_lat_pmax)?;
+
+        Ok(())
+    }
+}
+
+pub fn analyze(
+    duration: Duration,
+    iostat_start: &IoStat,
+    iostat_end: &IoStat,
+    metrics_dump_start: &MetricsDump,
+    metrics_dump_end: &MetricsDump,
+) -> Analysis {
+    let secs = duration.as_secs_f64();
+    let disk_read_iops = (iostat_end.read_ios - iostat_start.read_ios) as f64 / secs;
+    let disk_read_throughput =
+        (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+    let disk_write_iops = (iostat_end.write_ios - iostat_start.write_ios) as f64 / secs;
+    let disk_write_throughput =
+        (iostat_end.write_sectors - iostat_start.write_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+
+    let insert_iops = (metrics_dump_end.insert_ios - metrics_dump_start.insert_ios) as f64 / secs;
+    let insert_throughput =
+        (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
+
+    let get_iops = (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64 / secs;
+    let get_miss = (metrics_dump_end.get_miss_ios - metrics_dump_start.get_miss_ios) as f64
+        / (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64;
+    let get_throughput = (metrics_dump_end.get_bytes - metrics_dump_start.get_bytes) as f64 / secs;
+
+    Analysis {
+        disk_read_iops,
+        disk_read_throughput,
+        disk_write_iops,
+        disk_write_throughput,
+
+        insert_iops,
+        insert_throughput,
+        insert_lat_p50: metrics_dump_end.insert_lat_p50,
+        insert_lat_p90: metrics_dump_end.insert_lat_p90,
+        insert_lat_p99: metrics_dump_end.insert_lat_p99,
+        insert_lat_p999: metrics_dump_end.insert_lat_p999,
+        insert_lat_p9999: metrics_dump_end.insert_lat_p9999,
+        insert_lat_p99999: metrics_dump_end.insert_lat_p99999,
+        insert_lat_pmax: metrics_dump_end.insert_lat_pmax,
+
+        get_iops,
+        get_miss,
+        get_throughput,
+        get_hit_lat_p50: metrics_dump_end.get_hit_lat_p50,
+        get_hit_lat_p90: metrics_dump_end.get_hit_lat_p90,
+        get_hit_lat_p99: metrics_dump_end.get_hit_lat_p99,
+        get_hit_lat_p999: metrics_dump_end.get_hit_lat_p999,
+        get_hit_lat_p9999: metrics_dump_end.get_hit_lat_p9999,
+        get_hit_lat_p99999: metrics_dump_end.get_hit_lat_p99999,
+        get_hit_lat_pmax: metrics_dump_end.get_hit_lat_pmax,
+        get_miss_lat_p50: metrics_dump_end.get_miss_lat_p50,
+        get_miss_lat_p90: metrics_dump_end.get_miss_lat_p90,
+        get_miss_lat_p99: metrics_dump_end.get_miss_lat_p99,
+        get_miss_lat_p999: metrics_dump_end.get_miss_lat_p999,
+        get_miss_lat_p9999: metrics_dump_end.get_miss_lat_p9999,
+        get_miss_lat_p99999: metrics_dump_end.get_miss_lat_p99999,
+        get_miss_lat_pmax: metrics_dump_end.get_miss_lat_pmax,
+    }
+}
+
+pub async fn monitor(
+    iostat_path: impl AsRef<Path>,
+    interval: Duration,
+    total_secs: u64,
+    metrics: Metrics,
+    mut stop: broadcast::Receiver<()>,
+) {
+    let mut stat = iostat(&iostat_path);
+    let mut metrics_dump = metrics.dump();
+
+    let start = Instant::now();
+
+    loop {
+        let now = Instant::now();
+        match stop.try_recv() {
+            Err(broadcast::error::TryRecvError::Empty) => {}
+            _ => return,
+        }
+
+        tokio::time::sleep(interval).await;
+        let new_stat = iostat(&iostat_path);
+        let new_metrics_dump = metrics.dump();
+        let analysis = analyze(
+            // interval may have ~ +7% error
+            now.elapsed(),
+            &stat,
+            &new_stat,
+            &metrics_dump,
+            &new_metrics_dump,
+        );
+        println!("[{}s/{}s]", start.elapsed().as_secs(), total_secs);
+        println!("{}", analysis);
+        stat = new_stat;
+        metrics_dump = new_metrics_dump;
+    }
+}
diff --git a/foyer-experimental-bench/src/export.rs b/foyer-experimental-bench/src/export.rs
new file mode 100644
index 00000000..2d918221
--- /dev/null
+++ b/foyer-experimental-bench/src/export.rs
@@ -0,0 +1,70 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::net::SocketAddr;
+
+use foyer_storage::metrics::get_metrics_registry;
+use http_body_util::Full;
+use hyper::{
+    body::{Body, Bytes},
+    header::CONTENT_TYPE,
+    service::service_fn,
+    Request, Response,
+};
+use prometheus::{Encoder, TextEncoder};
+use tokio::net::TcpListener;
+
+pub struct MetricsExporter;
+
+impl MetricsExporter {
+    pub fn init(addr: SocketAddr) {
+        tokio::spawn(async move {
+            tracing::info!("Prometheus service is set up on http://{}", addr);
+
+            let listener = TcpListener::bind(addr).await.unwrap();
+            loop {
+                let stream = match listener.accept().await {
+                    Ok((stream, _addr)) => stream,
+                    Err(e) => {
+                        tracing::error!("accept conn error: {}", e);
+                        continue;
+                    }
+                };
+                let io = hyper_util::rt::TokioIo::new(stream);
+                tokio::spawn(async move {
+                    if let Err(e) = hyper_util::server::conn::auto::Builder::new(
+                        hyper_util::rt::TokioExecutor::new(),
+                    )
+                    .serve_connection(io, service_fn(Self::serve))
+                    .await
+                    {
+                        tracing::error!("Prometheus service error: {}", e);
+                    }
+                });
+            }
+        });
+    }
+
+    async fn serve(_request: Request<impl Body + Sized>) -> anyhow::Result<Response<Full<Bytes>>> {
+        let encoder = TextEncoder::new();
+        let mut buffer = Vec::with_capacity(4096);
+        let metrics = get_metrics_registry().gather();
+        encoder.encode(&metrics, &mut buffer)?;
+        let response = Response::builder()
+            .status(200)
+            .header(CONTENT_TYPE, encoder.format_type())
+            .body(Full::new(Bytes::from(buffer)))?;
+        Ok(response)
+    }
+}
diff --git a/foyer-experimental-bench/src/lib.rs b/foyer-experimental-bench/src/lib.rs
new file mode 100644
index 00000000..a4ee5955
--- /dev/null
+++ b/foyer-experimental-bench/src/lib.rs
@@ -0,0 +1,193 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+pub mod analyze;
+pub mod export;
+pub mod rate;
+pub mod text;
+pub mod utils;
+
+use std::{
+    path::PathBuf,
+    time::{Duration, Instant},
+};
+
+use analyze::{analyze, Metrics, MetricsDump};
+use tokio::task::JoinHandle;
+use utils::{iostat, IoStat};
+
+#[cfg(feature = "tokio-console")]
+pub fn init_logger() {
+    console_subscriber::init();
+}
+
+#[cfg(feature = "trace")]
+pub fn init_logger() {
+    use opentelemetry_sdk::{
+        trace::{BatchConfig, Config},
+        Resource,
+    };
+    use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
+    use tracing::Level;
+    use tracing_subscriber::{filter::Targets, prelude::*};
+
+    let trace_config =
+        Config::default().with_resource(Resource::new(vec![opentelemetry::KeyValue::new(
+            SERVICE_NAME,
+            "foyer-storage-bench",
+        )]));
+    let batch_config = BatchConfig::default()
+        .with_max_queue_size(1048576)
+        .with_max_export_batch_size(4096)
+        .with_max_concurrent_exports(4);
+
+    let tracer = opentelemetry_otlp::new_pipeline()
+        .tracing()
+        .with_exporter(opentelemetry_otlp::new_exporter().tonic())
+        .with_trace_config(trace_config)
+        .with_batch_config(batch_config)
+        .install_batch(opentelemetry_sdk::runtime::Tokio)
+        .unwrap();
+    let opentelemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer);
+    tracing_subscriber::registry()
+        .with(
+            Targets::new()
+                .with_target("foyer_storage", Level::DEBUG)
+                .with_target("foyer_common", Level::DEBUG)
+                .with_target("foyer_intrusive", Level::DEBUG)
+                .with_target("foyer_storage_bench", Level::DEBUG),
+        )
+        .with(opentelemetry_layer)
+        .init();
+}
+
+#[cfg(not(any(feature = "tokio-console", feature = "trace")))]
+pub fn init_logger() {
+    use tracing_subscriber::{prelude::*, EnvFilter};
+
+    tracing_subscriber::registry()
+        .with(
+            tracing_subscriber::fmt::layer()
+                // .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
+                .with_line_number(true),
+        )
+        .with(EnvFilter::from_default_env())
+        .init();
+}
+
+#[derive(Debug, Clone)]
+pub struct IoMonitorConfig {
+    pub dir: PathBuf,
+    pub interval: Duration,
+    pub total_secs: usize,
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn io_monitor(_: IoMonitorConfig) -> IoMonitorGuard {
+    panic!("bench tool io monitor only supports linux")
+}
+
+#[cfg(target_os = "linux")]
+pub fn io_monitor(config: IoMonitorConfig) -> IoMonitorGuard {
+    use utils::{detect_fs_type, io_stat_path, FsType};
+
+    let fs_type = detect_fs_type(&config.dir);
+    let io_stat_path = match fs_type {
+        FsType::Xfs | FsType::Ext4 => io_stat_path(&config.dir),
+        _ => panic!("bench tool io monitor only supports ext4 and xfs"),
+    };
+
+    let metrics = Metrics::default();
+
+    let start_io_stat = iostat(&io_stat_path);
+    let start_metrics_dump = metrics.dump();
+    let start = Instant::now();
+
+    let handle = {
+        let metrics = metrics.clone();
+        let config = config.clone();
+        let io_stat_path = io_stat_path.clone();
+        tokio::spawn(async move {
+            let mut last_io_stat = start_io_stat;
+            let mut last_metrics_dump = start_metrics_dump;
+            loop {
+                let now = Instant::now();
+                tokio::time::sleep(config.interval).await;
+                let new_io_stat = iostat(&io_stat_path);
+                let new_metrics_dump = metrics.dump();
+                let analysis = analyze(
+                    now.elapsed(),
+                    &last_io_stat,
+                    &new_io_stat,
+                    &last_metrics_dump,
+                    &new_metrics_dump,
+                );
+                println!(
+                    "Report [{}s/{}s]",
+                    start.elapsed().as_secs(),
+                    config.total_secs
+                );
+                println!("{}", analysis);
+                last_io_stat = new_io_stat;
+                last_metrics_dump = new_metrics_dump;
+            }
+        })
+    };
+
+    IoMonitorGuard {
+        config,
+        io_stat_path,
+
+        metrics,
+
+        start,
+        start_io_stat,
+        start_metrics_dump,
+
+        handle,
+    }
+}
+
+#[allow(dead_code)]
+#[must_use]
+pub struct IoMonitorGuard {
+    config: IoMonitorConfig,
+    io_stat_path: PathBuf,
+
+    metrics: Metrics,
+
+    start: Instant,
+    start_io_stat: IoStat,
+    start_metrics_dump: MetricsDump,
+
+    handle: JoinHandle<()>,
+}
+
+impl Drop for IoMonitorGuard {
+    fn drop(&mut self) {
+        self.handle.abort();
+        let end_io_stat = iostat(&self.io_stat_path);
+        let end_metrics_dump = self.metrics.dump();
+        let analysis = analyze(
+            self.start.elapsed(),
+            &self.start_io_stat,
+            &end_io_stat,
+            &self.start_metrics_dump,
+            &end_metrics_dump,
+        );
+        let elapsed = self.start.elapsed();
+        println!("Summary [{}s]", elapsed.as_secs());
+        println!("{}", analysis);
+    }
+}
diff --git a/foyer-experimental-bench/src/rate.rs b/foyer-experimental-bench/src/rate.rs
new file mode 100644
index 00000000..5be56a63
--- /dev/null
+++ b/foyer-experimental-bench/src/rate.rs
@@ -0,0 +1,59 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::{Duration, Instant};
+
+pub struct RateLimiter {
+    capacity: f64,
+    quota: f64,
+
+    last: Instant,
+}
+
+impl RateLimiter {
+    pub fn new(capacity: f64) -> Self {
+        Self {
+            capacity,
+            quota: 0.0,
+            last: Instant::now(),
+        }
+    }
+
+    pub fn consume(&mut self, weight: f64) -> Option<Duration> {
+        let now = Instant::now();
+        let refill = now.duration_since(self.last).as_secs_f64() * self.capacity;
+        self.last = now;
+        self.quota = f64::min(self.quota + refill, self.capacity);
+        self.quota -= weight;
+        if self.quota >= 0.0 {
+            return None;
+        }
+        let wait = Duration::from_secs_f64((-self.quota) / self.capacity);
+        Some(wait)
+    }
+}
diff --git a/foyer-experimental-bench/src/text.rs b/foyer-experimental-bench/src/text.rs
new file mode 100644
index 00000000..614d4365
--- /dev/null
+++ b/foyer-experimental-bench/src/text.rs
@@ -0,0 +1,28 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+const TEXT: &[u8] = include_bytes!("../etc/sample.txt");
+
+pub fn text(offset: usize, len: usize) -> Vec<u8> {
+    let mut res = Vec::with_capacity(len);
+    let mut cursor = offset % TEXT.len();
+    let mut remain = len;
+    while remain > 0 {
+        let bytes = std::cmp::min(remain, TEXT.len() - cursor);
+        res.extend(&TEXT[cursor..cursor + bytes]);
+        cursor = (cursor + bytes) % TEXT.len();
+        remain -= bytes;
+    }
+    res
+}
diff --git a/foyer-experimental-bench/src/utils.rs b/foyer-experimental-bench/src/utils.rs
new file mode 100644
index 00000000..2770a57c
--- /dev/null
+++ b/foyer-experimental-bench/src/utils.rs
@@ -0,0 +1,170 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::{Path, PathBuf};
+
+use itertools::Itertools;
+use nix::{fcntl::readlink, sys::stat::stat};
+
+#[cfg_attr(not(target_os = "linux"), expect(dead_code))]
+#[derive(PartialEq, Clone, Copy, Debug)]
+pub enum FsType {
+    Xfs,
+    Ext4,
+    Btrfs,
+    Tmpfs,
+    Others,
+}
+
+#[cfg_attr(not(target_os = "linux"), expect(unused_variables))]
+pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
+    #[cfg(target_os = "linux")]
+    {
+        use nix::sys::statfs::{
+            statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC,
+        };
+        let fs_stat = statfs(path.as_ref()).unwrap();
+        match fs_stat.filesystem_type() {
+            XFS_SUPER_MAGIC => FsType::Xfs,
+            EXT4_SUPER_MAGIC => FsType::Ext4,
+            BTRFS_SUPER_MAGIC => FsType::Btrfs,
+            TMPFS_MAGIC => FsType::Tmpfs,
+            _ => FsType::Others,
+        }
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    FsType::Others
+}
+
+/// Given a normal file path, returns the containing block device static file path (of the
+/// partition).
+pub fn io_stat_path(path: impl AsRef<Path>) -> PathBuf {
+    let st_dev = stat(path.as_ref()).unwrap().st_dev;
+
+    let major = unsafe { libc::major(st_dev) };
+    let minor = unsafe { libc::minor(st_dev) };
+
+    let dev = PathBuf::from("/dev/block").join(format!("{}:{}", major, minor));
+
+    let linkname = readlink(&dev).unwrap();
+    let devname = Path::new(linkname.as_os_str()).file_name().unwrap();
+    dev_stat_path(devname.to_str().unwrap())
+}
+
+pub fn dev_stat_path(devname: &str) -> PathBuf {
+    let classpath = Path::new("/sys/class/block").join(devname);
+    let devclass = readlink(&classpath).unwrap();
+
+    let devpath = Path::new(&devclass);
+    Path::new("/sys")
+        .join(devpath.strip_prefix("../..").unwrap())
+        .join("stat")
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct IoStat {
+    /// read I/Os       requests      number of read I/Os processed
+    pub read_ios: usize,
+    /// read merges     requests      number of read I/Os merged with in-queue I/O
+    pub read_merges: usize,
+    /// read sectors    sectors       number of sectors read
+    pub read_sectors: usize,
+    /// read ticks      milliseconds  total wait time for read requests
+    pub read_ticks: usize,
+    /// write I/Os      requests      number of write I/Os processed
+    pub write_ios: usize,
+    /// write merges    requests      number of write I/Os merged with in-queue I/O
+    pub write_merges: usize,
+    /// write sectors   sectors       number of sectors written
+    pub write_sectors: usize,
+    /// write ticks     milliseconds  total wait time for write requests
+    pub write_ticks: usize,
+    /// in_flight       requests      number of I/Os currently in flight
+    pub in_flight: usize,
+    /// io_ticks        milliseconds  total time this block device has been active
+    pub io_ticks: usize,
+    /// time_in_queue   milliseconds  total wait time for all requests
+    pub time_in_queue: usize,
+    /// discard I/Os    requests      number of discard I/Os processed
+    pub discard_ios: usize,
+    /// discard merges  requests      number of discard I/Os merged with in-queue I/O
+    pub discard_merges: usize,
+    /// discard sectors sectors       number of sectors discarded
+    pub discard_sectors: usize,
+    /// discard ticks   milliseconds  total wait time for discard requests
+    pub discard_ticks: usize,
+    /// flush I/Os      requests      number of flush I/Os processed
+    pub flush_ios: usize,
+    /// flush ticks     milliseconds  total wait time for flush requests
+    pub flush_ticks: usize,
+}
+
+/// Given the device static file path and get the iostat.
+pub fn iostat(path: impl AsRef<Path>) -> IoStat {
+    let content = std::fs::read_to_string(path.as_ref()).unwrap();
+    let nums = content.split_ascii_whitespace().collect_vec();
+
+    let read_ios = nums[0].parse().unwrap();
+    let read_merges = nums[1].parse().unwrap();
+    let read_sectors = nums[2].parse().unwrap();
+    let read_ticks = nums[3].parse().unwrap();
+    let write_ios = nums[4].parse().unwrap();
+    let write_merges = nums[5].parse().unwrap();
+    let write_sectors = nums[6].parse().unwrap();
+    let write_ticks = nums[7].parse().unwrap();
+    let in_flight = nums[8].parse().unwrap();
+    let io_ticks = nums[9].parse().unwrap();
+    let time_in_queue = nums[10].parse().unwrap();
+    let discard_ios = nums[11].parse().unwrap();
+    let discard_merges = nums[12].parse().unwrap();
+    let discard_sectors = nums[13].parse().unwrap();
+    let discard_ticks = nums[14].parse().unwrap();
+    let flush_ios = nums[15].parse().unwrap();
+    let flush_ticks = nums[16].parse().unwrap();
+
+    IoStat {
+        read_ios,
+        read_merges,
+        read_sectors,
+        read_ticks,
+        write_ios,
+        write_merges,
+        write_sectors,
+        write_ticks,
+        in_flight,
+        io_ticks,
+        time_in_queue,
+        discard_ios,
+        discard_merges,
+        discard_sectors,
+        discard_ticks,
+        flush_ios,
+        flush_ticks,
+    }
+}
diff --git a/foyer-experimental/Cargo.toml b/foyer-experimental/Cargo.toml
index aea40e21..889eac0d 100644
--- a/foyer-experimental/Cargo.toml
+++ b/foyer-experimental/Cargo.toml
@@ -19,6 +19,7 @@ crossbeam = { version = "0.8", features = ["crossbeam-channel"] }
 foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
+prometheus = "0.13"
 thiserror = "1"
 tokio = { workspace = true }
 tracing = "0.1"
@@ -35,7 +36,3 @@ tempfile = "3"
 
 [features]
 deadlock = ["parking_lot/deadlock_detection"]
-
-[[example]]
-name = "tombstone-log-bench"
-
diff --git a/foyer-experimental/src/lib.rs b/foyer-experimental/src/lib.rs
index f8d975ef..502141ae 100644
--- a/foyer-experimental/src/lib.rs
+++ b/foyer-experimental/src/lib.rs
@@ -15,10 +15,12 @@
 #![feature(cfg_match)]
 #![feature(lint_reasons)]
 #![feature(error_generic_member_access)]
+#![feature(lazy_cell)]
 
 pub mod buf;
 pub mod error;
-pub mod tombstone;
+pub mod metrics;
+pub mod wal;
 
 #[cfg(not(madsim))]
 #[tracing::instrument(level = "trace", skip(f))]
diff --git a/foyer-experimental/src/metrics.rs b/foyer-experimental/src/metrics.rs
new file mode 100644
index 00000000..d8860e11
--- /dev/null
+++ b/foyer-experimental/src/metrics.rs
@@ -0,0 +1,204 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::{LazyLock, OnceLock};
+
+use prometheus::{
+    core::{AtomicU64, GenericGauge, GenericGaugeVec},
+    exponential_buckets, opts, register_histogram_vec_with_registry,
+    register_int_counter_vec_with_registry, Histogram, HistogramVec, IntCounter, IntCounterVec,
+    Registry,
+};
+
+type UintGaugeVec = GenericGaugeVec<AtomicU64>;
+type UintGauge = GenericGauge<AtomicU64>;
+
+macro_rules! register_gauge_vec {
+    ($TYPE:ident, $OPTS:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
+        let gauge_vec = $TYPE::new($OPTS, $LABELS_NAMES).unwrap();
+        $REGISTRY
+            .register(Box::new(gauge_vec.clone()))
+            .map(|_| gauge_vec)
+    }};
+}
+
+macro_rules! register_uint_gauge_vec_with_registry {
+    ($OPTS:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
+        register_gauge_vec!(UintGaugeVec, $OPTS, $LABELS_NAMES, $REGISTRY)
+    }};
+
+    ($NAME:expr, $HELP:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
+        register_uint_gauge_vec_with_registry!(opts!($NAME, $HELP), $LABELS_NAMES, $REGISTRY)
+    }};
+}
+
+static REGISTRY: OnceLock<Registry> = OnceLock::new();
+
+/// Set metrics registry for `foyer`.
+///
+/// Metrics registry must be set before `open`.
+///
+/// Return `true` if set succeeds.
+pub fn set_metrics_registry(registry: Registry) -> bool {
+    REGISTRY.set(registry).is_ok()
+}
+
+pub fn get_metrics_registry() -> &'static Registry {
+    REGISTRY.get_or_init(|| prometheus::default_registry().clone())
+}
+
+pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
+
+#[derive(Debug)]
+pub struct GlobalMetrics {
+    pub op_duration: HistogramVec,
+    pub op_bytes: IntCounterVec,
+
+    pub total_bytes: UintGaugeVec,
+
+    pub inner_op_duration: HistogramVec,
+    pub inner_op_bytes: IntCounterVec,
+    pub inner_op_bytes_distribution: HistogramVec,
+}
+
+impl Default for GlobalMetrics {
+    fn default() -> Self {
+        Self::new(get_metrics_registry())
+    }
+}
+
+impl GlobalMetrics {
+    pub fn new(registry: &Registry) -> Self {
+        let op_duration = register_histogram_vec_with_registry!(
+            "foyer_storage_op_duration",
+            "foyer storage op duration",
+            &["foyer", "op", "extra"],
+            exponential_buckets(0.0001, 2.0, 16).unwrap(), // 100 us ~ 6.55 s
+            registry,
+        )
+        .unwrap();
+        let op_bytes = register_int_counter_vec_with_registry!(
+            "foyer_storage_op_bytes",
+            "foyer storage op bytes",
+            &["foyer", "op", "extra"],
+            registry,
+        )
+        .unwrap();
+
+        let total_bytes = register_uint_gauge_vec_with_registry!(
+            "foyer_storage_total_bytes",
+            "foyer storage total bytes",
+            &["foyer", "total", "extra"],
+            registry,
+        )
+        .unwrap();
+
+        let inner_op_duration = register_histogram_vec_with_registry!(
+            "foyer_storage_inner_op_duration",
+            "foyer storage inner op duration",
+            &["foyer", "op", "extra"],
+            exponential_buckets(0.0001, 2.0, 16).unwrap(), // 1 us ~ 0.655 s
+            registry,
+        )
+        .unwrap();
+        let inner_op_bytes = register_int_counter_vec_with_registry!(
+            "foyer_storage_inner_op_bytes",
+            "foyer storage inner op bytes",
+            &["foyer", "op", "extra"],
+            registry,
+        )
+        .unwrap();
+        let inner_op_bytes_distribution = register_histogram_vec_with_registry!(
+            "foyer_storage_inner_op_bytes_distribution",
+            "foyer storage inner op bytes distribution",
+            &["foyer", "op", "extra"],
+            exponential_buckets(0.0001, 2.0, 16).unwrap(), // 1 us ~ 0.655 s
+            registry,
+        )
+        .unwrap();
+
+        Self {
+            op_duration,
+            op_bytes,
+            total_bytes,
+            inner_op_duration,
+            inner_op_bytes,
+            inner_op_bytes_distribution,
+        }
+    }
+
+    pub fn metrics(&self, foyer: &str) -> Metrics {
+        Metrics::new(self, foyer)
+    }
+}
+
+#[derive(Debug)]
+pub struct Metrics {
+    pub inner_op_duration_wal_flush: Histogram,
+    pub inner_op_bytes_wal_flush: IntCounter,
+    pub inner_op_bytes_distribution_wal_flush: Histogram,
+
+    pub inner_op_duration_wal_write: Histogram,
+    pub inner_op_duration_wal_sync: Histogram,
+
+    pub inner_op_duration_wal_append: Histogram,
+    pub inner_op_duration_wal_notify: Histogram,
+
+    pub _total_bytes: UintGauge,
+}
+
+impl Metrics {
+    pub fn new(global: &GlobalMetrics, foyer: &str) -> Self {
+        let inner_op_duration_wal_flush = global
+            .inner_op_duration
+            .with_label_values(&[foyer, "wal", "flush"]);
+        let inner_op_bytes_wal_flush = global
+            .inner_op_bytes
+            .with_label_values(&[foyer, "wal", "flush"]);
+        let inner_op_bytes_distribution_wal_flush = global
+            .inner_op_bytes_distribution
+            .with_label_values(&[foyer, "wal", "flush"]);
+
+        let inner_op_duration_wal_write = global
+            .inner_op_duration
+            .with_label_values(&[foyer, "wal", "write"]);
+
+        let inner_op_duration_wal_sync = global
+            .inner_op_duration
+            .with_label_values(&[foyer, "wal", "sync"]);
+
+        let inner_op_duration_wal_append = global
+            .inner_op_duration
+            .with_label_values(&[foyer, "wal", "append"]);
+        let inner_op_duration_wal_notify = global
+            .inner_op_duration
+            .with_label_values(&[foyer, "wal", "notify"]);
+
+        let total_bytes = global.total_bytes.with_label_values(&[foyer, "", ""]);
+
+        Self {
+            inner_op_duration_wal_flush,
+            inner_op_bytes_wal_flush,
+            inner_op_bytes_distribution_wal_flush,
+
+            inner_op_duration_wal_write,
+            inner_op_duration_wal_sync,
+
+            inner_op_duration_wal_append,
+            inner_op_duration_wal_notify,
+
+            _total_bytes: total_bytes,
+        }
+    }
+}
diff --git a/foyer-experimental/src/tombstone.rs b/foyer-experimental/src/wal.rs
similarity index 86%
rename from foyer-experimental/src/tombstone.rs
rename to foyer-experimental/src/wal.rs
index d9f85661..0829e7e3 100644
--- a/foyer-experimental/src/tombstone.rs
+++ b/foyer-experimental/src/wal.rs
@@ -21,7 +21,6 @@ use std::{
         Arc,
     },
     thread::JoinHandle,
-    time::Instant,
 };
 
 use bytes::{Buf, BufMut};
@@ -33,6 +32,7 @@ use crate::{
     asyncify,
     buf::{BufExt, BufMutExt},
     error::Result,
+    metrics::Metrics,
 };
 
 pub trait HashValue: Send + Sync + 'static + Eq + std::fmt::Debug {
@@ -105,6 +105,7 @@ impl<H: HashValue> Tombstone<H> {
 pub struct TombstoneLogConfig {
     pub id: usize,
     pub dir: PathBuf,
+    pub metrics: Arc<Metrics>,
 }
 
 #[derive(Debug)]
@@ -122,6 +123,9 @@ struct TombstoneLogInner<H: HashValue> {
 #[derive(Debug)]
 pub struct TombstoneLog<H: HashValue> {
     inner: Arc<TombstoneLogInner<H>>,
+
+    metrics: Arc<Metrics>,
+
     stopped: Arc<AtomicBool>,
     handles: Arc<Mutex<Vec<JoinHandle<Result<()>>>>>,
 }
@@ -130,6 +134,7 @@ impl<H: HashValue> Clone for TombstoneLog<H> {
     fn clone(&self) -> Self {
         Self {
             inner: self.inner.clone(),
+            metrics: self.metrics.clone(),
             stopped: self.stopped.clone(),
             handles: self.handles.clone(),
         }
@@ -155,15 +160,20 @@ impl<H: HashValue> TombstoneLog<H> {
 
         let (task_tx, task_rx) = channel::unbounded();
         let stopped = Arc::new(AtomicBool::new(false));
+        let metrics = config.metrics.clone();
 
         let flusher = TombstoneLogFlusher {
             file,
             inner: inner.clone(),
             task_tx,
+            metrics: metrics.clone(),
             stopped: stopped.clone(),
         };
 
-        let notifier = TombstoneLogFlushNotifier { task_rx };
+        let notifier = TombstoneLogFlushNotifier {
+            task_rx,
+            metrics: metrics.clone(),
+        };
 
         let handles = vec![
             std::thread::spawn(move || flusher.run()),
@@ -173,6 +183,9 @@ impl<H: HashValue> TombstoneLog<H> {
 
         Ok(Self {
             inner,
+
+            metrics,
+
             stopped,
             handles,
         })
@@ -191,6 +204,8 @@ impl<H: HashValue> TombstoneLog<H> {
     }
 
     pub async fn append(&self, tombstone: Tombstone<H>) -> Result<()> {
+        let timer = self.metrics.inner_op_duration_wal_append.start_timer();
+
         let rx = {
             let mut inflights = self.inner.inflights.lock();
             let (tx, rx) = oneshot::channel();
@@ -198,15 +213,24 @@ impl<H: HashValue> TombstoneLog<H> {
             rx
         };
         self.inner.condvar.notify_one();
-        rx.await.unwrap()
+        let res = rx.await.unwrap();
+
+        drop(timer);
+
+        res
     }
 }
 
 #[derive(Debug)]
 struct TombstoneLogFlusher<H: HashValue> {
     file: File,
+
     inner: Arc<TombstoneLogInner<H>>,
+
     task_tx: channel::Sender<FlushNotifyTask>,
+
+    metrics: Arc<Metrics>,
+
     stopped: Arc<AtomicBool>,
 }
 
@@ -229,20 +253,17 @@ impl<H: HashValue> TombstoneLogFlusher<H> {
                 std::mem::take(&mut *inflights)
             };
 
-            let now = Instant::now();
+            let timer = self.metrics.inner_op_duration_wal_flush.start_timer();
 
             let mut buffer = Vec::with_capacity(Tombstone::<H>::size() * inflights.len());
             let mut txs = Vec::with_capacity(inflights.len());
-            let ts = inflights.len();
 
-            let write_buffer_start = Instant::now();
             for inflight in inflights {
                 inflight.tombstone.write(&mut buffer);
                 txs.push(inflight.tx);
             }
-            let write_buffer_duration = write_buffer_start.elapsed();
 
-            let write_all_start = Instant::now();
+            let timer_write = self.metrics.inner_op_duration_wal_write.start_timer();
             match self.file.write_all(&buffer) {
                 Ok(()) => {}
                 Err(e) => {
@@ -255,9 +276,9 @@ impl<H: HashValue> TombstoneLogFlusher<H> {
                     continue;
                 }
             }
-            let write_all_duration = write_all_start.elapsed();
+            drop(timer_write);
 
-            let fdatasync_start = Instant::now();
+            let timer_sync = self.metrics.inner_op_duration_wal_sync.start_timer();
             match self.file.sync_data() {
                 Ok(()) => {}
                 Err(e) => {
@@ -270,31 +291,16 @@ impl<H: HashValue> TombstoneLogFlusher<H> {
                     continue;
                 }
             }
-            let fdatasync_duration = fdatasync_start.elapsed();
+            drop(timer_sync);
 
-            let dispatch_start = Instant::now();
             self.task_tx
                 .send(FlushNotifyTask {
                     txs,
                     io_result: Ok(()),
                 })
                 .unwrap();
-            let dispatch_duration = dispatch_start.elapsed();
-
-            let duration = now.elapsed();
-
-            if duration.as_micros() >= 500 {
-                println!(
-                    "slow: {:?}, write buffer: {:?}, write all: {:?}, fdatasync: {:?}, dispatch: {:?}, ts: {:?}, buffer size: {:.3}KB",
-                    duration,
-                    write_buffer_duration,
-                    write_all_duration,
-                    fdatasync_duration,
-                    dispatch_duration,
-                    ts,
-                    buffer.len() as f64 / 4096.0,
-                );
-            }
+
+            drop(timer);
         }
     }
 }
@@ -308,11 +314,14 @@ struct FlushNotifyTask {
 #[derive(Debug)]
 struct TombstoneLogFlushNotifier {
     task_rx: channel::Receiver<FlushNotifyTask>,
+
+    metrics: Arc<Metrics>,
 }
 
 impl TombstoneLogFlushNotifier {
     fn run(self) -> Result<()> {
         while let Ok(task) = self.task_rx.recv() {
+            let timer = self.metrics.inner_op_duration_wal_notify.start_timer();
             for tx in task.txs {
                 let res = match &task.io_result {
                     Ok(()) => Ok(()),
@@ -320,6 +329,7 @@ impl TombstoneLogFlushNotifier {
                 };
                 tx.send(res).unwrap();
             }
+            drop(timer);
         }
         Ok(())
     }
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index ba4c4002..5620f72c 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -119,7 +119,6 @@ where
         {
             // only each `indices` holds one ref
             while region.refs().load(Ordering::SeqCst) > indices.len() {
-                println!("refs: {}", region.refs().load(Ordering::SeqCst));
                 tokio::time::sleep(Duration::from_millis(1)).await;
             }
         }

From d46ce2a731c6b4ccce250ea96e3e7c2aabffd86b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 23 Jan 2024 19:31:57 +0800
Subject: [PATCH 209/261] feat: introduce blocking notify (#260)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-experimental/Cargo.toml    |   2 +-
 foyer-experimental/src/lib.rs    |   1 +
 foyer-experimental/src/notify.rs | 129 +++++++++++++++++++++++++++++++
 foyer-experimental/src/wal.rs    |   3 -
 4 files changed, 131 insertions(+), 4 deletions(-)
 create mode 100644 foyer-experimental/src/notify.rs

diff --git a/foyer-experimental/Cargo.toml b/foyer-experimental/Cargo.toml
index 889eac0d..0101ae68 100644
--- a/foyer-experimental/Cargo.toml
+++ b/foyer-experimental/Cargo.toml
@@ -15,7 +15,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 anyhow = "1.0"
 bytes = "1"
-crossbeam = { version = "0.8", features = ["crossbeam-channel"] }
+crossbeam = { version = "0.8", features = ["std", "crossbeam-channel"] }
 foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
diff --git a/foyer-experimental/src/lib.rs b/foyer-experimental/src/lib.rs
index 502141ae..3233ff9f 100644
--- a/foyer-experimental/src/lib.rs
+++ b/foyer-experimental/src/lib.rs
@@ -20,6 +20,7 @@
 pub mod buf;
 pub mod error;
 pub mod metrics;
+pub mod notify;
 pub mod wal;
 
 #[cfg(not(madsim))]
diff --git a/foyer-experimental/src/notify.rs b/foyer-experimental/src/notify.rs
new file mode 100644
index 00000000..5c0b589a
--- /dev/null
+++ b/foyer-experimental/src/notify.rs
@@ -0,0 +1,129 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, OnceLock,
+    },
+    thread::{current, park, Thread},
+};
+
+use crossbeam::utils::Backoff;
+
+/// A multi-producer-single-consumer blocking notify implementation.
+#[derive(Debug, Default, Clone)]
+pub struct Notify {
+    ready: Arc<AtomicBool>,
+    thread: Arc<OnceLock<Thread>>,
+}
+
+impl Notify {
+    pub fn notified(&self) -> Notified {
+        self.thread.set(current()).unwrap();
+        Notified::new(self.ready.clone())
+    }
+
+    pub fn notify(&self) {
+        self.ready.store(true, Ordering::SeqCst);
+        self.thread.get().unwrap().unpark();
+    }
+}
+
+#[derive(Debug)]
+pub struct Notified {
+    ready: Arc<AtomicBool>,
+    backoff: Backoff,
+}
+
+impl Notified {
+    fn new(ready: Arc<AtomicBool>) -> Self {
+        Self {
+            ready,
+            backoff: Backoff::default(),
+        }
+    }
+
+    pub fn wait(&self) {
+        while !self.ready.load(Ordering::SeqCst) {
+            if self.backoff.is_completed() {
+                park();
+            } else {
+                self.backoff.snooze();
+            }
+        }
+        self.ready.store(false, Ordering::SeqCst);
+        self.backoff.reset();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        sync::Arc,
+        thread::{sleep, spawn},
+        time::Duration,
+    };
+
+    use super::*;
+
+    #[test]
+    fn assert_notify_send_sync_static() {
+        fn is_send_sync_static<T: Send + Sync + 'static>() {}
+        is_send_sync_static::<Notify>()
+    }
+
+    #[test]
+    fn test_notify_buffer_one() {
+        let notify = Notify::default();
+
+        let notified = notify.notified();
+
+        notify.notify();
+
+        notified.wait();
+    }
+
+    #[test]
+    fn test_notify_buffer_one_block() {
+        let handle = spawn(|| {
+            let notify = Notify::default();
+
+            let notified = notify.notified();
+
+            notify.notify();
+
+            notified.wait();
+            notified.wait();
+        });
+
+        sleep(Duration::from_secs(1));
+
+        assert!(!handle.is_finished());
+    }
+
+    #[test]
+    fn test_notify() {
+        let notify = Arc::new(Notify::default());
+        let n = notify.clone();
+        let handle = spawn(move || {
+            let notified = n.notified();
+
+            notified.wait();
+        });
+        sleep(Duration::from_secs(1));
+        notify.notify();
+        handle.join().unwrap();
+    }
+}
diff --git a/foyer-experimental/src/wal.rs b/foyer-experimental/src/wal.rs
index 0829e7e3..49c62839 100644
--- a/foyer-experimental/src/wal.rs
+++ b/foyer-experimental/src/wal.rs
@@ -334,6 +334,3 @@ impl TombstoneLogFlushNotifier {
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod tests {}

From 59d6a8854f0a9cc185709715d44a0b73e33aad52 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 28 Feb 2024 18:20:08 +0800
Subject: [PATCH 210/261] chore: fix typo, rename DList to Dlst to make it more
 Rustful (#271)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/collections/dlist.rs      | 122 ++++-----
 .../src/collections/duplicated_hashmap.rs     |  26 +-
 foyer-intrusive/src/collections/hashmap.rs    |  16 +-
 foyer-intrusive/src/core/adapter.rs           |  16 +-
 foyer-intrusive/src/eviction/fifo.rs          |  12 +-
 foyer-intrusive/src/eviction/lfu.rs           |  22 +-
 foyer-intrusive/src/eviction/lru.rs           |  18 +-
 foyer-intrusive/src/eviction/sfifo.rs         |  12 +-
 foyer-memory/src/eviction/fifo.rs             | 239 ++++++++++++++++++
 foyer-workspace-hack/Cargo.toml               |   1 +
 10 files changed, 362 insertions(+), 122 deletions(-)
 create mode 100644 foyer-memory/src/eviction/fifo.rs

diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index ddd8c8bc..93ed9b98 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -20,51 +20,51 @@ use crate::core::{
 };
 
 #[derive(Debug, Default)]
-pub struct DListLink {
-    prev: Option<NonNull<DListLink>>,
-    next: Option<NonNull<DListLink>>,
+pub struct DlistLink {
+    prev: Option<NonNull<DlistLink>>,
+    next: Option<NonNull<DlistLink>>,
     is_linked: bool,
 }
 
-impl DListLink {
-    pub fn raw(&self) -> NonNull<DListLink> {
+impl DlistLink {
+    pub fn raw(&self) -> NonNull<DlistLink> {
         unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
     }
 
-    pub fn prev(&self) -> Option<NonNull<DListLink>> {
+    pub fn prev(&self) -> Option<NonNull<DlistLink>> {
         self.prev
     }
 
-    pub fn next(&self) -> Option<NonNull<DListLink>> {
+    pub fn next(&self) -> Option<NonNull<DlistLink>> {
         self.next
     }
 }
 
-unsafe impl Send for DListLink {}
-unsafe impl Sync for DListLink {}
+unsafe impl Send for DlistLink {}
+unsafe impl Sync for DlistLink {}
 
-impl Link for DListLink {
+impl Link for DlistLink {
     fn is_linked(&self) -> bool {
         self.is_linked
     }
 }
 
 #[derive(Debug)]
-pub struct DList<A>
+pub struct Dlist<A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
-    head: Option<NonNull<DListLink>>,
-    tail: Option<NonNull<DListLink>>,
+    head: Option<NonNull<DlistLink>>,
+    tail: Option<NonNull<DlistLink>>,
 
     len: usize,
 
     adapter: A,
 }
 
-impl<A> Drop for DList<A>
+impl<A> Drop for Dlist<A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     fn drop(&mut self) {
         let mut iter = self.iter_mut();
@@ -76,9 +76,9 @@ where
     }
 }
 
-impl<A> DList<A>
+impl<A> Dlist<A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     pub fn new() -> Self {
         Self {
@@ -144,15 +144,15 @@ where
         iter.remove()
     }
 
-    pub fn iter(&self) -> DListIter<'_, A> {
-        DListIter {
+    pub fn iter(&self) -> DlistIter<'_, A> {
+        DlistIter {
             link: None,
             dlist: self,
         }
     }
 
-    pub fn iter_mut(&mut self) -> DListIterMut<'_, A> {
-        DListIterMut {
+    pub fn iter_mut(&mut self) -> DlistIterMut<'_, A> {
+        DlistIterMut {
             link: None,
             dlist: self,
         }
@@ -170,9 +170,9 @@ where
     ///
     /// # Safety
     ///
-    /// `link` MUST be in this [`DList`].
-    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DListLink>) -> DListIterMut<'_, A> {
-        DListIterMut {
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DlistLink>) -> DlistIterMut<'_, A> {
+        DlistIterMut {
             link: Some(link),
             dlist: self,
         }
@@ -182,9 +182,9 @@ where
     ///
     /// # Safety
     ///
-    /// `link` MUST be in this [`DList`].
-    pub unsafe fn iter_from_raw(&self, link: NonNull<DListLink>) -> DListIter<'_, A> {
-        DListIter {
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn iter_from_raw(&self, link: NonNull<DlistLink>) -> DlistIter<'_, A> {
+        DlistIter {
             link: Some(link),
             dlist: self,
         }
@@ -193,7 +193,7 @@ where
     /// # Safety
     ///
     /// `self` must be empty. `src` will be set empty after operation.
-    pub unsafe fn replace_with(&mut self, src: &mut DList<A>) {
+    pub unsafe fn replace_with(&mut self, src: &mut Dlist<A>) {
         debug_assert!(self.head.is_none());
         debug_assert!(self.tail.is_none());
         debug_assert_eq!(self.len, 0);
@@ -208,17 +208,17 @@ where
     }
 }
 
-pub struct DListIter<'a, A>
+pub struct DlistIter<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     link: Option<NonNull<A::Link>>,
-    dlist: &'a DList<A>,
+    dlist: &'a Dlist<A>,
 }
 
-impl<'a, A> DListIter<'a, A>
+impl<'a, A> DlistIter<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     pub fn is_valid(&self) -> bool {
         self.link.is_some()
@@ -274,17 +274,17 @@ where
     }
 }
 
-pub struct DListIterMut<'a, A>
+pub struct DlistIterMut<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     link: Option<NonNull<A::Link>>,
-    dlist: &'a mut DList<A>,
+    dlist: &'a mut Dlist<A>,
 }
 
-impl<'a, A> DListIterMut<'a, A>
+impl<'a, A> DlistIterMut<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     pub fn is_valid(&self) -> bool {
         self.link.is_some()
@@ -336,7 +336,7 @@ where
         self.link = self.dlist.tail;
     }
 
-    /// Removes the current item from [`DList`] and move next.
+    /// Removes the current item from [`Dlist`] and move next.
     pub fn remove(&mut self) -> Option<A::Pointer> {
         unsafe {
             if !self.is_valid() {
@@ -467,9 +467,9 @@ where
     }
 }
 
-impl<'a, A> Iterator for DListIter<'a, A>
+impl<'a, A> Iterator for DlistIter<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     type Item = &'a <A::Pointer as Pointer>::Item;
 
@@ -482,9 +482,9 @@ where
     }
 }
 
-impl<'a, A> Iterator for DListIterMut<'a, A>
+impl<'a, A> Iterator for DlistIterMut<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     type Item = &'a mut <A::Pointer as Pointer>::Item;
 
@@ -511,27 +511,27 @@ mod tests {
     use crate::intrusive_adapter;
 
     #[derive(Debug)]
-    struct DListItem {
-        link: DListLink,
+    struct DlistItem {
+        link: DlistLink,
         val: u64,
     }
 
-    impl DListItem {
+    impl DlistItem {
         fn new(val: u64) -> Self {
             Self {
-                link: DListLink::default(),
+                link: DlistLink::default(),
                 val,
             }
         }
     }
 
     #[derive(Debug, Default)]
-    struct DListAdapter;
+    struct DlistAdapter;
 
-    unsafe impl Adapter for DListAdapter {
-        type Pointer = Box<DListItem>;
+    unsafe impl Adapter for DlistAdapter {
+        type Pointer = Box<DlistItem>;
 
-        type Link = DListLink;
+        type Link = DlistLink;
 
         fn new() -> Self {
             Self
@@ -541,26 +541,26 @@ mod tests {
             &self,
             link: *const Self::Link,
         ) -> *const <Self::Pointer as Pointer>::Item {
-            crate::container_of!(link, DListItem, link)
+            crate::container_of!(link, DlistItem, link)
         }
 
         unsafe fn item2link(
             &self,
             item: *const <Self::Pointer as Pointer>::Item,
         ) -> *const Self::Link {
-            (item as *const u8).add(crate::offset_of!(DListItem, link)) as *const _
+            (item as *const u8).add(crate::offset_of!(DlistItem, link)) as *const _
         }
     }
 
-    intrusive_adapter! { DListArcAdapter = Arc<DListItem>: DListItem { link: DListLink } }
+    intrusive_adapter! { DlistArcAdapter = Arc<DlistItem>: DlistItem { link: DlistLink } }
 
     #[test]
     fn test_dlist_simple() {
-        let mut l = DList::<DListAdapter>::new();
+        let mut l = Dlist::<DlistAdapter>::new();
 
-        l.push_back(Box::new(DListItem::new(2)));
-        l.push_front(Box::new(DListItem::new(1)));
-        l.push_back(Box::new(DListItem::new(3)));
+        l.push_back(Box::new(DlistItem::new(2)));
+        l.push_front(Box::new(DlistItem::new(1)));
+        l.push_back(Box::new(DlistItem::new(3)));
 
         let v = l.iter_mut().map(|item| item.val).collect_vec();
         assert_eq!(v, vec![1, 2, 3]);
@@ -587,9 +587,9 @@ mod tests {
 
     #[test]
     fn test_arc_drop() {
-        let mut l = DList::<DListArcAdapter>::new();
+        let mut l = Dlist::<DlistArcAdapter>::new();
 
-        let items = (0..10).map(|i| Arc::new(DListItem::new(i))).collect_vec();
+        let items = (0..10).map(|i| Arc::new(DlistItem::new(i))).collect_vec();
         for item in items.iter() {
             l.push_back(item.clone());
         }
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index 49c2494b..16593a68 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -17,7 +17,7 @@ use std::{fmt::Debug, hash::Hasher, marker::PhantomData, ptr::NonNull};
 use foyer_common::code::{Key, Value};
 use twox_hash::XxHash64;
 
-use super::dlist::{DList, DListIter, DListIterMut, DListLink};
+use super::dlist::{Dlist, DlistIter, DlistIterMut, DlistLink};
 use crate::{
     core::{
         adapter::{Adapter, KeyAdapter, Link},
@@ -27,17 +27,17 @@ use crate::{
 };
 
 pub struct DuplicatedHashMapLink {
-    slot_link: DListLink,
-    group_link: DListLink,
-    group: DList<DuplicatedHashMapLinkGroupAdapter>,
+    slot_link: DlistLink,
+    group_link: DlistLink,
+    group: Dlist<DuplicatedHashMapLinkGroupAdapter>,
 }
 
 impl Default for DuplicatedHashMapLink {
     fn default() -> Self {
         Self {
-            slot_link: DListLink::default(),
-            group_link: DListLink::default(),
-            group: DList::new(),
+            slot_link: DlistLink::default(),
+            group_link: DlistLink::default(),
+            group: Dlist::new(),
         }
     }
 }
@@ -51,8 +51,8 @@ impl Debug for DuplicatedHashMapLink {
     }
 }
 
-intrusive_adapter! { DuplicatedHashMapLinkSlotAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { slot_link: DListLink } }
-intrusive_adapter! { DuplicatedHashMapLinkGroupAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { group_link: DListLink } }
+intrusive_adapter! { DuplicatedHashMapLinkSlotAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { slot_link: DlistLink } }
+intrusive_adapter! { DuplicatedHashMapLinkGroupAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { group_link: DlistLink } }
 
 impl DuplicatedHashMapLink {
     pub fn raw(&self) -> NonNull<DuplicatedHashMapLink> {
@@ -75,7 +75,7 @@ where
     V: Value,
     A: KeyAdapter<Key = K, Link = DuplicatedHashMapLink>,
 {
-    slots: Vec<DList<DuplicatedHashMapLinkSlotAdapter>>,
+    slots: Vec<Dlist<DuplicatedHashMapLinkSlotAdapter>>,
 
     len: usize,
 
@@ -119,7 +119,7 @@ where
     pub fn new(bits: usize) -> Self {
         let mut slots = Vec::with_capacity(1 << bits);
         for _ in 0..(1 << bits) {
-            slots.push(DList::new());
+            slots.push(Dlist::new());
         }
         Self {
             slots,
@@ -284,7 +284,7 @@ where
         &mut self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIterMut<'_, DuplicatedHashMapLinkSlotAdapter>> {
+    ) -> Option<DlistIterMut<'_, DuplicatedHashMapLinkSlotAdapter>> {
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
@@ -305,7 +305,7 @@ where
         &self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
+    ) -> Option<DlistIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 961d0566..8e715e17 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -17,7 +17,7 @@ use std::{hash::Hasher, marker::PhantomData, ptr::NonNull};
 use foyer_common::code::{Key, Value};
 use twox_hash::XxHash64;
 
-use super::dlist::{DList, DListIter, DListIterMut, DListLink};
+use super::dlist::{Dlist, DlistIter, DlistIterMut, DlistLink};
 use crate::{
     core::{
         adapter::{KeyAdapter, Link},
@@ -28,10 +28,10 @@ use crate::{
 
 #[derive(Debug, Default)]
 pub struct HashMapLink {
-    dlist_link: DListLink,
+    dlist_link: DlistLink,
 }
 
-intrusive_adapter! { HashMapLinkAdapter = NonNull<HashMapLink>: HashMapLink { dlist_link: DListLink } }
+intrusive_adapter! { HashMapLinkAdapter = NonNull<HashMapLink>: HashMapLink { dlist_link: DlistLink } }
 
 impl HashMapLink {
     pub fn raw(&self) -> NonNull<HashMapLink> {
@@ -54,7 +54,7 @@ where
     V: Value,
     A: KeyAdapter<Key = K, Link = HashMapLink>,
 {
-    slots: Vec<DList<HashMapLinkAdapter>>,
+    slots: Vec<Dlist<HashMapLinkAdapter>>,
 
     len: usize,
 
@@ -93,7 +93,7 @@ where
     pub fn new(bits: usize) -> Self {
         let mut slots = Vec::with_capacity(1 << bits);
         for _ in 0..(1 << bits) {
-            slots.push(DList::new());
+            slots.push(Dlist::new());
         }
         Self {
             slots,
@@ -189,7 +189,7 @@ where
         &mut self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIterMut<'_, HashMapLinkAdapter>> {
+    ) -> Option<DlistIterMut<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
@@ -210,7 +210,7 @@ where
         &self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIter<'_, HashMapLinkAdapter>> {
+    ) -> Option<DlistIter<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
@@ -253,7 +253,7 @@ where
     A: KeyAdapter<Key = K, Link = HashMapLink>,
 {
     slot: usize,
-    iters: Vec<DListIter<'a, HashMapLinkAdapter>>,
+    iters: Vec<DlistIter<'a, HashMapLinkAdapter>>,
 
     adapter: A,
     _marker: PhantomData<(K, V)>,
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 32057323..4dcaf275 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -331,27 +331,27 @@ mod tests {
     use crate::{collections::dlist::*, intrusive_adapter};
 
     #[derive(Debug)]
-    struct DListItem {
-        link: DListLink,
+    struct DlistItem {
+        link: DlistLink,
         val: u64,
     }
 
-    impl DListItem {
+    impl DlistItem {
         fn new(val: u64) -> Self {
             Self {
-                link: DListLink::default(),
+                link: DlistLink::default(),
                 val,
             }
         }
     }
 
-    intrusive_adapter! { DListItemAdapter = Box<DListItem>: DListItem { link: DListLink }}
-    key_adapter! { DListItemAdapter = DListItem { val: u64 } }
+    intrusive_adapter! { DlistItemAdapter = Box<DlistItem>: DlistItem { link: DlistLink }}
+    key_adapter! { DlistItemAdapter = DlistItem { val: u64 } }
 
     #[test]
     fn test_adapter_macro() {
-        let mut l = DList::<DListItemAdapter>::new();
-        l.push_front(Box::new(DListItem::new(1)));
+        let mut l = Dlist::<DlistItemAdapter>::new();
+        l.push_front(Box::new(DlistItem::new(1)));
         let v = l.iter().map(|item| item.val).collect_vec();
         assert_eq!(v, vec![1]);
     }
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 494b9b4e..2a97c5aa 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -30,7 +30,7 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, Link},
         pointer::Pointer,
@@ -43,7 +43,7 @@ pub struct FifoConfig;
 
 #[derive(Debug, Default)]
 pub struct FifoLink {
-    link: DListLink,
+    link: DlistLink,
 }
 
 impl FifoLink {
@@ -58,7 +58,7 @@ impl Link for FifoLink {
     }
 }
 
-intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DListLink } }
+intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DlistLink } }
 
 /// FIFO policy
 #[derive(Debug)]
@@ -67,7 +67,7 @@ where
     A: Adapter<Link = FifoLink>,
     <A as Adapter>::Pointer: Clone,
 {
-    queue: DList<FifoLinkAdapter>,
+    queue: Dlist<FifoLinkAdapter>,
 
     _config: FifoConfig,
 
@@ -99,7 +99,7 @@ where
 {
     pub fn new(config: FifoConfig) -> Self {
         Self {
-            queue: DList::new(),
+            queue: Dlist::new(),
 
             _config: config,
 
@@ -165,7 +165,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     fifo: &'a Fifo<A>,
-    iter: DListIter<'a, FifoLinkAdapter>,
+    iter: DlistIter<'a, FifoLinkAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 6893fab2..fbb01684 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -37,7 +37,7 @@ use twox_hash::XxHash64;
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, KeyAdapter, Link},
         pointer::Pointer,
@@ -69,8 +69,8 @@ enum LruType {
 
 #[derive(Debug, Default)]
 pub struct LfuLink {
-    link_tiny: DListLink,
-    link_main: DListLink,
+    link_tiny: DlistLink,
+    link_main: DlistLink,
 }
 
 impl Link for LfuLink {
@@ -94,8 +94,8 @@ impl LfuLink {
     }
 }
 
-intrusive_adapter! { LfuLinkTinyDListAdapter = NonNull<LfuLink>: LfuLink { link_tiny: DListLink } }
-intrusive_adapter! { LfuLinkMainDListAdapter = NonNull<LfuLink>: LfuLink { link_main: DListLink } }
+intrusive_adapter! { LfuLinkTinyDlistAdapter = NonNull<LfuLink>: LfuLink { link_tiny: DlistLink } }
+intrusive_adapter! { LfuLinkMainDlistAdapter = NonNull<LfuLink>: LfuLink { link_main: DlistLink } }
 
 /// Implements the W-TinyLFU cache eviction policy as described in -
 ///
@@ -131,10 +131,10 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     /// tiny lru list
-    lru_tiny: DList<LfuLinkTinyDListAdapter>,
+    lru_tiny: Dlist<LfuLinkTinyDlistAdapter>,
 
     /// main lru list
-    lru_main: DList<LfuLinkMainDListAdapter>,
+    lru_main: Dlist<LfuLinkMainDlistAdapter>,
 
     /// the window length counter
     window_size: usize,
@@ -180,8 +180,8 @@ where
 {
     pub fn new(config: LfuConfig) -> Self {
         let mut res = Self {
-            lru_tiny: DList::new(),
-            lru_main: DList::new(),
+            lru_tiny: Dlist::new(),
+            lru_main: Dlist::new(),
 
             window_size: 0,
             max_window_size: 0,
@@ -415,8 +415,8 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     lfu: &'a Lfu<A>,
-    iter_tiny: DListIter<'a, LfuLinkTinyDListAdapter>,
-    iter_main: DListIter<'a, LfuLinkMainDListAdapter>,
+    iter_tiny: DlistIter<'a, LfuLinkTinyDlistAdapter>,
+    iter_main: DlistIter<'a, LfuLinkMainDlistAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 2915b0e5..dba2e8de 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -30,7 +30,7 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, Link},
         pointer::Pointer,
@@ -46,7 +46,7 @@ pub struct LruConfig {
 
 #[derive(Debug, Default)]
 pub struct LruLink {
-    link_lru: DListLink,
+    link_lru: DlistLink,
 
     is_in_tail: bool,
 }
@@ -63,7 +63,7 @@ impl Link for LruLink {
     }
 }
 
-intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DListLink } }
+intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DlistLink } }
 
 #[derive(Debug)]
 pub struct Lru<A>
@@ -72,7 +72,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     /// lru list
-    lru: DList<LruLinkAdapter>,
+    lru: Dlist<LruLinkAdapter>,
 
     /// insertion point
     insertion_point: Option<NonNull<LruLink>>,
@@ -110,7 +110,7 @@ where
 {
     fn new(config: LruConfig) -> Self {
         Self {
-            lru: DList::new(),
+            lru: Dlist::new(),
 
             insertion_point: None,
 
@@ -146,7 +146,7 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            self.ensuer_not_insertion_point(link);
+            self.ensure_not_insertion_point(link);
             self.lru
                 .iter_mut_from_raw(link.as_ref().link_lru.raw())
                 .remove()
@@ -169,7 +169,7 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            self.ensuer_not_insertion_point(link);
+            self.ensure_not_insertion_point(link);
 
             self.move_to_lru_front(link);
 
@@ -239,7 +239,7 @@ where
         }
     }
 
-    unsafe fn ensuer_not_insertion_point(&mut self, link: NonNull<LruLink>) {
+    unsafe fn ensure_not_insertion_point(&mut self, link: NonNull<LruLink>) {
         if Some(link) == self.insertion_point {
             self.insertion_point = self.lru_prev(link);
             match &mut self.insertion_point {
@@ -290,7 +290,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     lru: &'a Lru<A>,
-    iter: DListIter<'a, LruLinkAdapter>,
+    iter: DlistIter<'a, LruLinkAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index a7dca2ad..2fea2bad 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -32,7 +32,7 @@ use itertools::Itertools;
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, Link, PriorityAdapter},
         pointer::Pointer,
@@ -52,7 +52,7 @@ pub struct SegmentedFifoConfig {
 
 #[derive(Debug, Default)]
 pub struct SegmentedFifoLink {
-    link_queue: DListLink,
+    link_queue: DlistLink,
 
     priority: usize,
 }
@@ -69,7 +69,7 @@ impl Link for SegmentedFifoLink {
     }
 }
 
-intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: SegmentedFifoLink { link_queue: DListLink } }
+intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: SegmentedFifoLink { link_queue: DlistLink } }
 
 /// Segmented FIFO policy
 ///
@@ -109,7 +109,7 @@ where
     A::Pointer: Clone,
 {
     // Note: All queue share the same dlist link.
-    segments: Vec<DList<SegmentedFifoLinkAdapter>>,
+    segments: Vec<Dlist<SegmentedFifoLinkAdapter>>,
 
     config: SegmentedFifoConfig,
     total_ratio: usize,
@@ -142,7 +142,7 @@ where
 {
     pub fn new(config: SegmentedFifoConfig) -> Self {
         let segments = (0..config.segment_ratios.len())
-            .map(|_| DList::new())
+            .map(|_| Dlist::new())
             .collect_vec();
         let total_ratio = config.segment_ratios.iter().sum();
 
@@ -249,7 +249,7 @@ where
     A::Pointer: Clone,
 {
     sfifo: &'a SegmentedFifo<A>,
-    iter_segments: Vec<DListIter<'a, SegmentedFifoLinkAdapter>>,
+    iter_segments: Vec<DlistIter<'a, SegmentedFifoLinkAdapter>>,
     segment: usize,
 
     ptr: ManuallyDrop<Option<A::Pointer>>,
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
new file mode 100644
index 00000000..6caab705
--- /dev/null
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -0,0 +1,239 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, hash::BuildHasher, ptr::NonNull};
+
+use foyer_intrusive::{
+    collections::dlist::{Dlist, DlistLink},
+    intrusive_adapter,
+};
+
+use crate::{
+    cache::CacheConfig,
+    eviction::Eviction,
+    handle::{BaseHandle, Handle},
+    Key, Value,
+};
+
+pub type FifoContext = ();
+
+pub struct FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    link: DlistLink,
+    base: BaseHandle<K, V, FifoContext>,
+}
+
+impl<K, V> Debug for FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("FifoHandle").finish()
+    }
+}
+
+intrusive_adapter! { FifoHandleDlistAdapter<K,V> = NonNull<FifoHandle<K,V>>: FifoHandle<K,V> { link: DlistLink } where K:Key, V:Value }
+
+impl<K, V> Handle for FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Context = FifoContext;
+
+    fn new() -> Self {
+        Self {
+            link: DlistLink::default(),
+            base: BaseHandle::new(),
+        }
+    }
+
+    fn init(
+        &mut self,
+        hash: u64,
+        key: Self::Key,
+        value: Self::Value,
+        charge: usize,
+        context: Self::Context,
+    ) {
+        self.base.init(hash, key, value, charge, context);
+    }
+
+    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &self.base
+    }
+
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &mut self.base
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FifoConfig {}
+
+pub struct Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    dlist: Dlist<FifoHandleDlistAdapter<K, V>>,
+}
+
+impl<K, V> Eviction for Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Handle = FifoHandle<K, V>;
+    type Config = FifoConfig;
+
+    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(_: &CacheConfig<Self, S>) -> Self
+    where
+        Self: Sized,
+    {
+        Self {
+            dlist: Dlist::new(),
+        }
+    }
+
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+        self.dlist.push_back(ptr);
+        ptr.as_mut().base_mut().set_in_eviction(true);
+    }
+
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+        self.dlist.pop_front().map(|mut ptr| {
+            ptr.as_mut().base_mut().set_in_eviction(false);
+            ptr
+        })
+    }
+
+    unsafe fn reinsert(&mut self, _: NonNull<Self::Handle>) -> bool {
+        false
+    }
+
+    unsafe fn access(&mut self, _: NonNull<Self::Handle>) {}
+
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let p = self
+            .dlist
+            .iter_mut_from_raw(ptr.as_mut().link.raw())
+            .remove()
+            .unwrap();
+        assert_eq!(p, ptr);
+        ptr.as_mut().base_mut().set_in_eviction(false);
+    }
+
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+        let mut res = Vec::with_capacity(self.len());
+        while let Some(mut ptr) = self.dlist.pop_front() {
+            ptr.as_mut().base_mut().set_in_eviction(false);
+            res.push(ptr);
+        }
+        res
+    }
+
+    unsafe fn len(&self) -> usize {
+        self.dlist.len()
+    }
+
+    unsafe fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+unsafe impl<K, V> Send for Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+unsafe impl<K, V> Sync for Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+
+#[cfg(test)]
+mod tests {
+    use ahash::RandomState;
+    use itertools::Itertools;
+
+    use super::*;
+
+    type TestFifoHandle = FifoHandle<u64, u64>;
+    type TestFifo = Fifo<u64, u64>;
+
+    unsafe fn new_test_fifo_handle_ptr(key: u64, value: u64) -> NonNull<TestFifoHandle> {
+        let mut handle = Box::new(TestFifoHandle::new());
+        handle.init(0, key, value, 1, ());
+        NonNull::new_unchecked(Box::into_raw(handle))
+    }
+
+    unsafe fn del_test_fifo_handle_ptr(ptr: NonNull<TestFifoHandle>) {
+        let _ = Box::from_raw(ptr.as_ptr());
+    }
+
+    #[test]
+    fn test_fifo() {
+        unsafe {
+            let ptrs = (0..8).map(|i| new_test_fifo_handle_ptr(i, i)).collect_vec();
+
+            let config = CacheConfig {
+                capacity: 0,
+                shards: 1,
+                eviction_config: FifoConfig {},
+                object_pool_capacity: 0,
+                hash_builder: RandomState::default(),
+            };
+
+            let mut fifo = TestFifo::new(&config);
+
+            // 0, 1, 2, 3
+            fifo.push(ptrs[0]);
+            fifo.push(ptrs[1]);
+            fifo.push(ptrs[2]);
+            fifo.push(ptrs[3]);
+
+            // 2, 3
+            let p0 = fifo.pop().unwrap();
+            let p1 = fifo.pop().unwrap();
+            assert_eq!(ptrs[0], p0);
+            assert_eq!(ptrs[1], p1);
+
+            // 2, 3, 4, 5, 6
+            fifo.push(ptrs[4]);
+            fifo.push(ptrs[5]);
+            fifo.push(ptrs[6]);
+
+            // 2, 6
+            fifo.remove(ptrs[3]);
+            fifo.remove(ptrs[4]);
+            fifo.remove(ptrs[5]);
+
+            assert_eq!(fifo.clear(), vec![ptrs[2], ptrs[6]]);
+
+            for ptr in ptrs {
+                del_test_fifo_handle_ptr(ptr);
+            }
+        }
+    }
+}
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 1d436b89..2b30e528 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -30,6 +30,7 @@ memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
+smallvec = { version = "1", default-features = false, features = ["const_new"] }
 tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 tracing-core = { version = "0.1" }
 

From 78c4d9d1a531a4d53df8626bc4da0e0acfc830b1 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 29 Feb 2024 11:28:53 +0800
Subject: [PATCH 211/261] Revert "chore: fix typo, rename DList to Dlst to make
 it more Rustful" (#272)

This reverts commit 59d6a8854f0a9cc185709715d44a0b73e33aad52.
---
 foyer-intrusive/src/collections/dlist.rs      | 122 ++++-----
 .../src/collections/duplicated_hashmap.rs     |  26 +-
 foyer-intrusive/src/collections/hashmap.rs    |  16 +-
 foyer-intrusive/src/core/adapter.rs           |  16 +-
 foyer-intrusive/src/eviction/fifo.rs          |  12 +-
 foyer-intrusive/src/eviction/lfu.rs           |  22 +-
 foyer-intrusive/src/eviction/lru.rs           |  18 +-
 foyer-intrusive/src/eviction/sfifo.rs         |  12 +-
 foyer-memory/src/eviction/fifo.rs             | 239 ------------------
 9 files changed, 122 insertions(+), 361 deletions(-)
 delete mode 100644 foyer-memory/src/eviction/fifo.rs

diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index 93ed9b98..ddd8c8bc 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -20,51 +20,51 @@ use crate::core::{
 };
 
 #[derive(Debug, Default)]
-pub struct DlistLink {
-    prev: Option<NonNull<DlistLink>>,
-    next: Option<NonNull<DlistLink>>,
+pub struct DListLink {
+    prev: Option<NonNull<DListLink>>,
+    next: Option<NonNull<DListLink>>,
     is_linked: bool,
 }
 
-impl DlistLink {
-    pub fn raw(&self) -> NonNull<DlistLink> {
+impl DListLink {
+    pub fn raw(&self) -> NonNull<DListLink> {
         unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
     }
 
-    pub fn prev(&self) -> Option<NonNull<DlistLink>> {
+    pub fn prev(&self) -> Option<NonNull<DListLink>> {
         self.prev
     }
 
-    pub fn next(&self) -> Option<NonNull<DlistLink>> {
+    pub fn next(&self) -> Option<NonNull<DListLink>> {
         self.next
     }
 }
 
-unsafe impl Send for DlistLink {}
-unsafe impl Sync for DlistLink {}
+unsafe impl Send for DListLink {}
+unsafe impl Sync for DListLink {}
 
-impl Link for DlistLink {
+impl Link for DListLink {
     fn is_linked(&self) -> bool {
         self.is_linked
     }
 }
 
 #[derive(Debug)]
-pub struct Dlist<A>
+pub struct DList<A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
-    head: Option<NonNull<DlistLink>>,
-    tail: Option<NonNull<DlistLink>>,
+    head: Option<NonNull<DListLink>>,
+    tail: Option<NonNull<DListLink>>,
 
     len: usize,
 
     adapter: A,
 }
 
-impl<A> Drop for Dlist<A>
+impl<A> Drop for DList<A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     fn drop(&mut self) {
         let mut iter = self.iter_mut();
@@ -76,9 +76,9 @@ where
     }
 }
 
-impl<A> Dlist<A>
+impl<A> DList<A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     pub fn new() -> Self {
         Self {
@@ -144,15 +144,15 @@ where
         iter.remove()
     }
 
-    pub fn iter(&self) -> DlistIter<'_, A> {
-        DlistIter {
+    pub fn iter(&self) -> DListIter<'_, A> {
+        DListIter {
             link: None,
             dlist: self,
         }
     }
 
-    pub fn iter_mut(&mut self) -> DlistIterMut<'_, A> {
-        DlistIterMut {
+    pub fn iter_mut(&mut self) -> DListIterMut<'_, A> {
+        DListIterMut {
             link: None,
             dlist: self,
         }
@@ -170,9 +170,9 @@ where
     ///
     /// # Safety
     ///
-    /// `link` MUST be in this [`Dlist`].
-    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DlistLink>) -> DlistIterMut<'_, A> {
-        DlistIterMut {
+    /// `link` MUST be in this [`DList`].
+    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DListLink>) -> DListIterMut<'_, A> {
+        DListIterMut {
             link: Some(link),
             dlist: self,
         }
@@ -182,9 +182,9 @@ where
     ///
     /// # Safety
     ///
-    /// `link` MUST be in this [`Dlist`].
-    pub unsafe fn iter_from_raw(&self, link: NonNull<DlistLink>) -> DlistIter<'_, A> {
-        DlistIter {
+    /// `link` MUST be in this [`DList`].
+    pub unsafe fn iter_from_raw(&self, link: NonNull<DListLink>) -> DListIter<'_, A> {
+        DListIter {
             link: Some(link),
             dlist: self,
         }
@@ -193,7 +193,7 @@ where
     /// # Safety
     ///
     /// `self` must be empty. `src` will be set empty after operation.
-    pub unsafe fn replace_with(&mut self, src: &mut Dlist<A>) {
+    pub unsafe fn replace_with(&mut self, src: &mut DList<A>) {
         debug_assert!(self.head.is_none());
         debug_assert!(self.tail.is_none());
         debug_assert_eq!(self.len, 0);
@@ -208,17 +208,17 @@ where
     }
 }
 
-pub struct DlistIter<'a, A>
+pub struct DListIter<'a, A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     link: Option<NonNull<A::Link>>,
-    dlist: &'a Dlist<A>,
+    dlist: &'a DList<A>,
 }
 
-impl<'a, A> DlistIter<'a, A>
+impl<'a, A> DListIter<'a, A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     pub fn is_valid(&self) -> bool {
         self.link.is_some()
@@ -274,17 +274,17 @@ where
     }
 }
 
-pub struct DlistIterMut<'a, A>
+pub struct DListIterMut<'a, A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     link: Option<NonNull<A::Link>>,
-    dlist: &'a mut Dlist<A>,
+    dlist: &'a mut DList<A>,
 }
 
-impl<'a, A> DlistIterMut<'a, A>
+impl<'a, A> DListIterMut<'a, A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     pub fn is_valid(&self) -> bool {
         self.link.is_some()
@@ -336,7 +336,7 @@ where
         self.link = self.dlist.tail;
     }
 
-    /// Removes the current item from [`Dlist`] and move next.
+    /// Removes the current item from [`DList`] and move next.
     pub fn remove(&mut self) -> Option<A::Pointer> {
         unsafe {
             if !self.is_valid() {
@@ -467,9 +467,9 @@ where
     }
 }
 
-impl<'a, A> Iterator for DlistIter<'a, A>
+impl<'a, A> Iterator for DListIter<'a, A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     type Item = &'a <A::Pointer as Pointer>::Item;
 
@@ -482,9 +482,9 @@ where
     }
 }
 
-impl<'a, A> Iterator for DlistIterMut<'a, A>
+impl<'a, A> Iterator for DListIterMut<'a, A>
 where
-    A: Adapter<Link = DlistLink>,
+    A: Adapter<Link = DListLink>,
 {
     type Item = &'a mut <A::Pointer as Pointer>::Item;
 
@@ -511,27 +511,27 @@ mod tests {
     use crate::intrusive_adapter;
 
     #[derive(Debug)]
-    struct DlistItem {
-        link: DlistLink,
+    struct DListItem {
+        link: DListLink,
         val: u64,
     }
 
-    impl DlistItem {
+    impl DListItem {
         fn new(val: u64) -> Self {
             Self {
-                link: DlistLink::default(),
+                link: DListLink::default(),
                 val,
             }
         }
     }
 
     #[derive(Debug, Default)]
-    struct DlistAdapter;
+    struct DListAdapter;
 
-    unsafe impl Adapter for DlistAdapter {
-        type Pointer = Box<DlistItem>;
+    unsafe impl Adapter for DListAdapter {
+        type Pointer = Box<DListItem>;
 
-        type Link = DlistLink;
+        type Link = DListLink;
 
         fn new() -> Self {
             Self
@@ -541,26 +541,26 @@ mod tests {
             &self,
             link: *const Self::Link,
         ) -> *const <Self::Pointer as Pointer>::Item {
-            crate::container_of!(link, DlistItem, link)
+            crate::container_of!(link, DListItem, link)
         }
 
         unsafe fn item2link(
             &self,
             item: *const <Self::Pointer as Pointer>::Item,
         ) -> *const Self::Link {
-            (item as *const u8).add(crate::offset_of!(DlistItem, link)) as *const _
+            (item as *const u8).add(crate::offset_of!(DListItem, link)) as *const _
         }
     }
 
-    intrusive_adapter! { DlistArcAdapter = Arc<DlistItem>: DlistItem { link: DlistLink } }
+    intrusive_adapter! { DListArcAdapter = Arc<DListItem>: DListItem { link: DListLink } }
 
     #[test]
     fn test_dlist_simple() {
-        let mut l = Dlist::<DlistAdapter>::new();
+        let mut l = DList::<DListAdapter>::new();
 
-        l.push_back(Box::new(DlistItem::new(2)));
-        l.push_front(Box::new(DlistItem::new(1)));
-        l.push_back(Box::new(DlistItem::new(3)));
+        l.push_back(Box::new(DListItem::new(2)));
+        l.push_front(Box::new(DListItem::new(1)));
+        l.push_back(Box::new(DListItem::new(3)));
 
         let v = l.iter_mut().map(|item| item.val).collect_vec();
         assert_eq!(v, vec![1, 2, 3]);
@@ -587,9 +587,9 @@ mod tests {
 
     #[test]
     fn test_arc_drop() {
-        let mut l = Dlist::<DlistArcAdapter>::new();
+        let mut l = DList::<DListArcAdapter>::new();
 
-        let items = (0..10).map(|i| Arc::new(DlistItem::new(i))).collect_vec();
+        let items = (0..10).map(|i| Arc::new(DListItem::new(i))).collect_vec();
         for item in items.iter() {
             l.push_back(item.clone());
         }
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index 16593a68..49c2494b 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -17,7 +17,7 @@ use std::{fmt::Debug, hash::Hasher, marker::PhantomData, ptr::NonNull};
 use foyer_common::code::{Key, Value};
 use twox_hash::XxHash64;
 
-use super::dlist::{Dlist, DlistIter, DlistIterMut, DlistLink};
+use super::dlist::{DList, DListIter, DListIterMut, DListLink};
 use crate::{
     core::{
         adapter::{Adapter, KeyAdapter, Link},
@@ -27,17 +27,17 @@ use crate::{
 };
 
 pub struct DuplicatedHashMapLink {
-    slot_link: DlistLink,
-    group_link: DlistLink,
-    group: Dlist<DuplicatedHashMapLinkGroupAdapter>,
+    slot_link: DListLink,
+    group_link: DListLink,
+    group: DList<DuplicatedHashMapLinkGroupAdapter>,
 }
 
 impl Default for DuplicatedHashMapLink {
     fn default() -> Self {
         Self {
-            slot_link: DlistLink::default(),
-            group_link: DlistLink::default(),
-            group: Dlist::new(),
+            slot_link: DListLink::default(),
+            group_link: DListLink::default(),
+            group: DList::new(),
         }
     }
 }
@@ -51,8 +51,8 @@ impl Debug for DuplicatedHashMapLink {
     }
 }
 
-intrusive_adapter! { DuplicatedHashMapLinkSlotAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { slot_link: DlistLink } }
-intrusive_adapter! { DuplicatedHashMapLinkGroupAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { group_link: DlistLink } }
+intrusive_adapter! { DuplicatedHashMapLinkSlotAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { slot_link: DListLink } }
+intrusive_adapter! { DuplicatedHashMapLinkGroupAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { group_link: DListLink } }
 
 impl DuplicatedHashMapLink {
     pub fn raw(&self) -> NonNull<DuplicatedHashMapLink> {
@@ -75,7 +75,7 @@ where
     V: Value,
     A: KeyAdapter<Key = K, Link = DuplicatedHashMapLink>,
 {
-    slots: Vec<Dlist<DuplicatedHashMapLinkSlotAdapter>>,
+    slots: Vec<DList<DuplicatedHashMapLinkSlotAdapter>>,
 
     len: usize,
 
@@ -119,7 +119,7 @@ where
     pub fn new(bits: usize) -> Self {
         let mut slots = Vec::with_capacity(1 << bits);
         for _ in 0..(1 << bits) {
-            slots.push(Dlist::new());
+            slots.push(DList::new());
         }
         Self {
             slots,
@@ -284,7 +284,7 @@ where
         &mut self,
         key: &K,
         slot: usize,
-    ) -> Option<DlistIterMut<'_, DuplicatedHashMapLinkSlotAdapter>> {
+    ) -> Option<DListIterMut<'_, DuplicatedHashMapLinkSlotAdapter>> {
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
@@ -305,7 +305,7 @@ where
         &self,
         key: &K,
         slot: usize,
-    ) -> Option<DlistIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
+    ) -> Option<DListIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 8e715e17..961d0566 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -17,7 +17,7 @@ use std::{hash::Hasher, marker::PhantomData, ptr::NonNull};
 use foyer_common::code::{Key, Value};
 use twox_hash::XxHash64;
 
-use super::dlist::{Dlist, DlistIter, DlistIterMut, DlistLink};
+use super::dlist::{DList, DListIter, DListIterMut, DListLink};
 use crate::{
     core::{
         adapter::{KeyAdapter, Link},
@@ -28,10 +28,10 @@ use crate::{
 
 #[derive(Debug, Default)]
 pub struct HashMapLink {
-    dlist_link: DlistLink,
+    dlist_link: DListLink,
 }
 
-intrusive_adapter! { HashMapLinkAdapter = NonNull<HashMapLink>: HashMapLink { dlist_link: DlistLink } }
+intrusive_adapter! { HashMapLinkAdapter = NonNull<HashMapLink>: HashMapLink { dlist_link: DListLink } }
 
 impl HashMapLink {
     pub fn raw(&self) -> NonNull<HashMapLink> {
@@ -54,7 +54,7 @@ where
     V: Value,
     A: KeyAdapter<Key = K, Link = HashMapLink>,
 {
-    slots: Vec<Dlist<HashMapLinkAdapter>>,
+    slots: Vec<DList<HashMapLinkAdapter>>,
 
     len: usize,
 
@@ -93,7 +93,7 @@ where
     pub fn new(bits: usize) -> Self {
         let mut slots = Vec::with_capacity(1 << bits);
         for _ in 0..(1 << bits) {
-            slots.push(Dlist::new());
+            slots.push(DList::new());
         }
         Self {
             slots,
@@ -189,7 +189,7 @@ where
         &mut self,
         key: &K,
         slot: usize,
-    ) -> Option<DlistIterMut<'_, HashMapLinkAdapter>> {
+    ) -> Option<DListIterMut<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
@@ -210,7 +210,7 @@ where
         &self,
         key: &K,
         slot: usize,
-    ) -> Option<DlistIter<'_, HashMapLinkAdapter>> {
+    ) -> Option<DListIter<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
@@ -253,7 +253,7 @@ where
     A: KeyAdapter<Key = K, Link = HashMapLink>,
 {
     slot: usize,
-    iters: Vec<DlistIter<'a, HashMapLinkAdapter>>,
+    iters: Vec<DListIter<'a, HashMapLinkAdapter>>,
 
     adapter: A,
     _marker: PhantomData<(K, V)>,
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 4dcaf275..32057323 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -331,27 +331,27 @@ mod tests {
     use crate::{collections::dlist::*, intrusive_adapter};
 
     #[derive(Debug)]
-    struct DlistItem {
-        link: DlistLink,
+    struct DListItem {
+        link: DListLink,
         val: u64,
     }
 
-    impl DlistItem {
+    impl DListItem {
         fn new(val: u64) -> Self {
             Self {
-                link: DlistLink::default(),
+                link: DListLink::default(),
                 val,
             }
         }
     }
 
-    intrusive_adapter! { DlistItemAdapter = Box<DlistItem>: DlistItem { link: DlistLink }}
-    key_adapter! { DlistItemAdapter = DlistItem { val: u64 } }
+    intrusive_adapter! { DListItemAdapter = Box<DListItem>: DListItem { link: DListLink }}
+    key_adapter! { DListItemAdapter = DListItem { val: u64 } }
 
     #[test]
     fn test_adapter_macro() {
-        let mut l = Dlist::<DlistItemAdapter>::new();
-        l.push_front(Box::new(DlistItem::new(1)));
+        let mut l = DList::<DListItemAdapter>::new();
+        l.push_front(Box::new(DListItem::new(1)));
         let v = l.iter().map(|item| item.val).collect_vec();
         assert_eq!(v, vec![1]);
     }
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 2a97c5aa..494b9b4e 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -30,7 +30,7 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{Dlist, DlistIter, DlistLink},
+    collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, Link},
         pointer::Pointer,
@@ -43,7 +43,7 @@ pub struct FifoConfig;
 
 #[derive(Debug, Default)]
 pub struct FifoLink {
-    link: DlistLink,
+    link: DListLink,
 }
 
 impl FifoLink {
@@ -58,7 +58,7 @@ impl Link for FifoLink {
     }
 }
 
-intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DlistLink } }
+intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DListLink } }
 
 /// FIFO policy
 #[derive(Debug)]
@@ -67,7 +67,7 @@ where
     A: Adapter<Link = FifoLink>,
     <A as Adapter>::Pointer: Clone,
 {
-    queue: Dlist<FifoLinkAdapter>,
+    queue: DList<FifoLinkAdapter>,
 
     _config: FifoConfig,
 
@@ -99,7 +99,7 @@ where
 {
     pub fn new(config: FifoConfig) -> Self {
         Self {
-            queue: Dlist::new(),
+            queue: DList::new(),
 
             _config: config,
 
@@ -165,7 +165,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     fifo: &'a Fifo<A>,
-    iter: DlistIter<'a, FifoLinkAdapter>,
+    iter: DListIter<'a, FifoLinkAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index fbb01684..6893fab2 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -37,7 +37,7 @@ use twox_hash::XxHash64;
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{Dlist, DlistIter, DlistLink},
+    collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, KeyAdapter, Link},
         pointer::Pointer,
@@ -69,8 +69,8 @@ enum LruType {
 
 #[derive(Debug, Default)]
 pub struct LfuLink {
-    link_tiny: DlistLink,
-    link_main: DlistLink,
+    link_tiny: DListLink,
+    link_main: DListLink,
 }
 
 impl Link for LfuLink {
@@ -94,8 +94,8 @@ impl LfuLink {
     }
 }
 
-intrusive_adapter! { LfuLinkTinyDlistAdapter = NonNull<LfuLink>: LfuLink { link_tiny: DlistLink } }
-intrusive_adapter! { LfuLinkMainDlistAdapter = NonNull<LfuLink>: LfuLink { link_main: DlistLink } }
+intrusive_adapter! { LfuLinkTinyDListAdapter = NonNull<LfuLink>: LfuLink { link_tiny: DListLink } }
+intrusive_adapter! { LfuLinkMainDListAdapter = NonNull<LfuLink>: LfuLink { link_main: DListLink } }
 
 /// Implements the W-TinyLFU cache eviction policy as described in -
 ///
@@ -131,10 +131,10 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     /// tiny lru list
-    lru_tiny: Dlist<LfuLinkTinyDlistAdapter>,
+    lru_tiny: DList<LfuLinkTinyDListAdapter>,
 
     /// main lru list
-    lru_main: Dlist<LfuLinkMainDlistAdapter>,
+    lru_main: DList<LfuLinkMainDListAdapter>,
 
     /// the window length counter
     window_size: usize,
@@ -180,8 +180,8 @@ where
 {
     pub fn new(config: LfuConfig) -> Self {
         let mut res = Self {
-            lru_tiny: Dlist::new(),
-            lru_main: Dlist::new(),
+            lru_tiny: DList::new(),
+            lru_main: DList::new(),
 
             window_size: 0,
             max_window_size: 0,
@@ -415,8 +415,8 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     lfu: &'a Lfu<A>,
-    iter_tiny: DlistIter<'a, LfuLinkTinyDlistAdapter>,
-    iter_main: DlistIter<'a, LfuLinkMainDlistAdapter>,
+    iter_tiny: DListIter<'a, LfuLinkTinyDListAdapter>,
+    iter_main: DListIter<'a, LfuLinkMainDListAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index dba2e8de..2915b0e5 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -30,7 +30,7 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{Dlist, DlistIter, DlistLink},
+    collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, Link},
         pointer::Pointer,
@@ -46,7 +46,7 @@ pub struct LruConfig {
 
 #[derive(Debug, Default)]
 pub struct LruLink {
-    link_lru: DlistLink,
+    link_lru: DListLink,
 
     is_in_tail: bool,
 }
@@ -63,7 +63,7 @@ impl Link for LruLink {
     }
 }
 
-intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DlistLink } }
+intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DListLink } }
 
 #[derive(Debug)]
 pub struct Lru<A>
@@ -72,7 +72,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     /// lru list
-    lru: Dlist<LruLinkAdapter>,
+    lru: DList<LruLinkAdapter>,
 
     /// insertion point
     insertion_point: Option<NonNull<LruLink>>,
@@ -110,7 +110,7 @@ where
 {
     fn new(config: LruConfig) -> Self {
         Self {
-            lru: Dlist::new(),
+            lru: DList::new(),
 
             insertion_point: None,
 
@@ -146,7 +146,7 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            self.ensure_not_insertion_point(link);
+            self.ensuer_not_insertion_point(link);
             self.lru
                 .iter_mut_from_raw(link.as_ref().link_lru.raw())
                 .remove()
@@ -169,7 +169,7 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            self.ensure_not_insertion_point(link);
+            self.ensuer_not_insertion_point(link);
 
             self.move_to_lru_front(link);
 
@@ -239,7 +239,7 @@ where
         }
     }
 
-    unsafe fn ensure_not_insertion_point(&mut self, link: NonNull<LruLink>) {
+    unsafe fn ensuer_not_insertion_point(&mut self, link: NonNull<LruLink>) {
         if Some(link) == self.insertion_point {
             self.insertion_point = self.lru_prev(link);
             match &mut self.insertion_point {
@@ -290,7 +290,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     lru: &'a Lru<A>,
-    iter: DlistIter<'a, LruLinkAdapter>,
+    iter: DListIter<'a, LruLinkAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index 2fea2bad..a7dca2ad 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -32,7 +32,7 @@ use itertools::Itertools;
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{Dlist, DlistIter, DlistLink},
+    collections::dlist::{DList, DListIter, DListLink},
     core::{
         adapter::{Adapter, Link, PriorityAdapter},
         pointer::Pointer,
@@ -52,7 +52,7 @@ pub struct SegmentedFifoConfig {
 
 #[derive(Debug, Default)]
 pub struct SegmentedFifoLink {
-    link_queue: DlistLink,
+    link_queue: DListLink,
 
     priority: usize,
 }
@@ -69,7 +69,7 @@ impl Link for SegmentedFifoLink {
     }
 }
 
-intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: SegmentedFifoLink { link_queue: DlistLink } }
+intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: SegmentedFifoLink { link_queue: DListLink } }
 
 /// Segmented FIFO policy
 ///
@@ -109,7 +109,7 @@ where
     A::Pointer: Clone,
 {
     // Note: All queue share the same dlist link.
-    segments: Vec<Dlist<SegmentedFifoLinkAdapter>>,
+    segments: Vec<DList<SegmentedFifoLinkAdapter>>,
 
     config: SegmentedFifoConfig,
     total_ratio: usize,
@@ -142,7 +142,7 @@ where
 {
     pub fn new(config: SegmentedFifoConfig) -> Self {
         let segments = (0..config.segment_ratios.len())
-            .map(|_| Dlist::new())
+            .map(|_| DList::new())
             .collect_vec();
         let total_ratio = config.segment_ratios.iter().sum();
 
@@ -249,7 +249,7 @@ where
     A::Pointer: Clone,
 {
     sfifo: &'a SegmentedFifo<A>,
-    iter_segments: Vec<DlistIter<'a, SegmentedFifoLinkAdapter>>,
+    iter_segments: Vec<DListIter<'a, SegmentedFifoLinkAdapter>>,
     segment: usize,
 
     ptr: ManuallyDrop<Option<A::Pointer>>,
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
deleted file mode 100644
index 6caab705..00000000
--- a/foyer-memory/src/eviction/fifo.rs
+++ /dev/null
@@ -1,239 +0,0 @@
-//  Copyright 2024 MrCroxx
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{fmt::Debug, hash::BuildHasher, ptr::NonNull};
-
-use foyer_intrusive::{
-    collections::dlist::{Dlist, DlistLink},
-    intrusive_adapter,
-};
-
-use crate::{
-    cache::CacheConfig,
-    eviction::Eviction,
-    handle::{BaseHandle, Handle},
-    Key, Value,
-};
-
-pub type FifoContext = ();
-
-pub struct FifoHandle<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    link: DlistLink,
-    base: BaseHandle<K, V, FifoContext>,
-}
-
-impl<K, V> Debug for FifoHandle<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("FifoHandle").finish()
-    }
-}
-
-intrusive_adapter! { FifoHandleDlistAdapter<K,V> = NonNull<FifoHandle<K,V>>: FifoHandle<K,V> { link: DlistLink } where K:Key, V:Value }
-
-impl<K, V> Handle for FifoHandle<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Key = K;
-    type Value = V;
-    type Context = FifoContext;
-
-    fn new() -> Self {
-        Self {
-            link: DlistLink::default(),
-            base: BaseHandle::new(),
-        }
-    }
-
-    fn init(
-        &mut self,
-        hash: u64,
-        key: Self::Key,
-        value: Self::Value,
-        charge: usize,
-        context: Self::Context,
-    ) {
-        self.base.init(hash, key, value, charge, context);
-    }
-
-    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
-        &self.base
-    }
-
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
-        &mut self.base
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct FifoConfig {}
-
-pub struct Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    dlist: Dlist<FifoHandleDlistAdapter<K, V>>,
-}
-
-impl<K, V> Eviction for Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Handle = FifoHandle<K, V>;
-    type Config = FifoConfig;
-
-    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(_: &CacheConfig<Self, S>) -> Self
-    where
-        Self: Sized,
-    {
-        Self {
-            dlist: Dlist::new(),
-        }
-    }
-
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
-        self.dlist.push_back(ptr);
-        ptr.as_mut().base_mut().set_in_eviction(true);
-    }
-
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
-        self.dlist.pop_front().map(|mut ptr| {
-            ptr.as_mut().base_mut().set_in_eviction(false);
-            ptr
-        })
-    }
-
-    unsafe fn reinsert(&mut self, _: NonNull<Self::Handle>) -> bool {
-        false
-    }
-
-    unsafe fn access(&mut self, _: NonNull<Self::Handle>) {}
-
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let p = self
-            .dlist
-            .iter_mut_from_raw(ptr.as_mut().link.raw())
-            .remove()
-            .unwrap();
-        assert_eq!(p, ptr);
-        ptr.as_mut().base_mut().set_in_eviction(false);
-    }
-
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
-        let mut res = Vec::with_capacity(self.len());
-        while let Some(mut ptr) = self.dlist.pop_front() {
-            ptr.as_mut().base_mut().set_in_eviction(false);
-            res.push(ptr);
-        }
-        res
-    }
-
-    unsafe fn len(&self) -> usize {
-        self.dlist.len()
-    }
-
-    unsafe fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-}
-
-unsafe impl<K, V> Send for Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-unsafe impl<K, V> Sync for Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-
-#[cfg(test)]
-mod tests {
-    use ahash::RandomState;
-    use itertools::Itertools;
-
-    use super::*;
-
-    type TestFifoHandle = FifoHandle<u64, u64>;
-    type TestFifo = Fifo<u64, u64>;
-
-    unsafe fn new_test_fifo_handle_ptr(key: u64, value: u64) -> NonNull<TestFifoHandle> {
-        let mut handle = Box::new(TestFifoHandle::new());
-        handle.init(0, key, value, 1, ());
-        NonNull::new_unchecked(Box::into_raw(handle))
-    }
-
-    unsafe fn del_test_fifo_handle_ptr(ptr: NonNull<TestFifoHandle>) {
-        let _ = Box::from_raw(ptr.as_ptr());
-    }
-
-    #[test]
-    fn test_fifo() {
-        unsafe {
-            let ptrs = (0..8).map(|i| new_test_fifo_handle_ptr(i, i)).collect_vec();
-
-            let config = CacheConfig {
-                capacity: 0,
-                shards: 1,
-                eviction_config: FifoConfig {},
-                object_pool_capacity: 0,
-                hash_builder: RandomState::default(),
-            };
-
-            let mut fifo = TestFifo::new(&config);
-
-            // 0, 1, 2, 3
-            fifo.push(ptrs[0]);
-            fifo.push(ptrs[1]);
-            fifo.push(ptrs[2]);
-            fifo.push(ptrs[3]);
-
-            // 2, 3
-            let p0 = fifo.pop().unwrap();
-            let p1 = fifo.pop().unwrap();
-            assert_eq!(ptrs[0], p0);
-            assert_eq!(ptrs[1], p1);
-
-            // 2, 3, 4, 5, 6
-            fifo.push(ptrs[4]);
-            fifo.push(ptrs[5]);
-            fifo.push(ptrs[6]);
-
-            // 2, 6
-            fifo.remove(ptrs[3]);
-            fifo.remove(ptrs[4]);
-            fifo.remove(ptrs[5]);
-
-            assert_eq!(fifo.clear(), vec![ptrs[2], ptrs[6]]);
-
-            for ptr in ptrs {
-                del_test_fifo_handle_ptr(ptr);
-            }
-        }
-    }
-}

From aea4d394990d85773699bafee0feaadaa15af9e7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 29 Feb 2024 11:34:31 +0800
Subject: [PATCH 212/261] chore: fix typo, rename DList to Dlst to make it more
 Rustful (#273)

* chore: fix typo, rename DList to Dlst to make it more Rustful

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove files that doesn't belong to this PR

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/collections/dlist.rs      | 122 +++++++++---------
 .../src/collections/duplicated_hashmap.rs     |  26 ++--
 foyer-intrusive/src/collections/hashmap.rs    |  16 +--
 foyer-intrusive/src/core/adapter.rs           |  16 +--
 foyer-intrusive/src/eviction/fifo.rs          |  12 +-
 foyer-intrusive/src/eviction/lfu.rs           |  22 ++--
 foyer-intrusive/src/eviction/lru.rs           |  18 +--
 foyer-intrusive/src/eviction/sfifo.rs         |  12 +-
 8 files changed, 122 insertions(+), 122 deletions(-)

diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index ddd8c8bc..93ed9b98 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -20,51 +20,51 @@ use crate::core::{
 };
 
 #[derive(Debug, Default)]
-pub struct DListLink {
-    prev: Option<NonNull<DListLink>>,
-    next: Option<NonNull<DListLink>>,
+pub struct DlistLink {
+    prev: Option<NonNull<DlistLink>>,
+    next: Option<NonNull<DlistLink>>,
     is_linked: bool,
 }
 
-impl DListLink {
-    pub fn raw(&self) -> NonNull<DListLink> {
+impl DlistLink {
+    pub fn raw(&self) -> NonNull<DlistLink> {
         unsafe { NonNull::new_unchecked(self as *const _ as *mut _) }
     }
 
-    pub fn prev(&self) -> Option<NonNull<DListLink>> {
+    pub fn prev(&self) -> Option<NonNull<DlistLink>> {
         self.prev
     }
 
-    pub fn next(&self) -> Option<NonNull<DListLink>> {
+    pub fn next(&self) -> Option<NonNull<DlistLink>> {
         self.next
     }
 }
 
-unsafe impl Send for DListLink {}
-unsafe impl Sync for DListLink {}
+unsafe impl Send for DlistLink {}
+unsafe impl Sync for DlistLink {}
 
-impl Link for DListLink {
+impl Link for DlistLink {
     fn is_linked(&self) -> bool {
         self.is_linked
     }
 }
 
 #[derive(Debug)]
-pub struct DList<A>
+pub struct Dlist<A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
-    head: Option<NonNull<DListLink>>,
-    tail: Option<NonNull<DListLink>>,
+    head: Option<NonNull<DlistLink>>,
+    tail: Option<NonNull<DlistLink>>,
 
     len: usize,
 
     adapter: A,
 }
 
-impl<A> Drop for DList<A>
+impl<A> Drop for Dlist<A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     fn drop(&mut self) {
         let mut iter = self.iter_mut();
@@ -76,9 +76,9 @@ where
     }
 }
 
-impl<A> DList<A>
+impl<A> Dlist<A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     pub fn new() -> Self {
         Self {
@@ -144,15 +144,15 @@ where
         iter.remove()
     }
 
-    pub fn iter(&self) -> DListIter<'_, A> {
-        DListIter {
+    pub fn iter(&self) -> DlistIter<'_, A> {
+        DlistIter {
             link: None,
             dlist: self,
         }
     }
 
-    pub fn iter_mut(&mut self) -> DListIterMut<'_, A> {
-        DListIterMut {
+    pub fn iter_mut(&mut self) -> DlistIterMut<'_, A> {
+        DlistIterMut {
             link: None,
             dlist: self,
         }
@@ -170,9 +170,9 @@ where
     ///
     /// # Safety
     ///
-    /// `link` MUST be in this [`DList`].
-    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DListLink>) -> DListIterMut<'_, A> {
-        DListIterMut {
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DlistLink>) -> DlistIterMut<'_, A> {
+        DlistIterMut {
             link: Some(link),
             dlist: self,
         }
@@ -182,9 +182,9 @@ where
     ///
     /// # Safety
     ///
-    /// `link` MUST be in this [`DList`].
-    pub unsafe fn iter_from_raw(&self, link: NonNull<DListLink>) -> DListIter<'_, A> {
-        DListIter {
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn iter_from_raw(&self, link: NonNull<DlistLink>) -> DlistIter<'_, A> {
+        DlistIter {
             link: Some(link),
             dlist: self,
         }
@@ -193,7 +193,7 @@ where
     /// # Safety
     ///
     /// `self` must be empty. `src` will be set empty after operation.
-    pub unsafe fn replace_with(&mut self, src: &mut DList<A>) {
+    pub unsafe fn replace_with(&mut self, src: &mut Dlist<A>) {
         debug_assert!(self.head.is_none());
         debug_assert!(self.tail.is_none());
         debug_assert_eq!(self.len, 0);
@@ -208,17 +208,17 @@ where
     }
 }
 
-pub struct DListIter<'a, A>
+pub struct DlistIter<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     link: Option<NonNull<A::Link>>,
-    dlist: &'a DList<A>,
+    dlist: &'a Dlist<A>,
 }
 
-impl<'a, A> DListIter<'a, A>
+impl<'a, A> DlistIter<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     pub fn is_valid(&self) -> bool {
         self.link.is_some()
@@ -274,17 +274,17 @@ where
     }
 }
 
-pub struct DListIterMut<'a, A>
+pub struct DlistIterMut<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     link: Option<NonNull<A::Link>>,
-    dlist: &'a mut DList<A>,
+    dlist: &'a mut Dlist<A>,
 }
 
-impl<'a, A> DListIterMut<'a, A>
+impl<'a, A> DlistIterMut<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     pub fn is_valid(&self) -> bool {
         self.link.is_some()
@@ -336,7 +336,7 @@ where
         self.link = self.dlist.tail;
     }
 
-    /// Removes the current item from [`DList`] and move next.
+    /// Removes the current item from [`Dlist`] and move next.
     pub fn remove(&mut self) -> Option<A::Pointer> {
         unsafe {
             if !self.is_valid() {
@@ -467,9 +467,9 @@ where
     }
 }
 
-impl<'a, A> Iterator for DListIter<'a, A>
+impl<'a, A> Iterator for DlistIter<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     type Item = &'a <A::Pointer as Pointer>::Item;
 
@@ -482,9 +482,9 @@ where
     }
 }
 
-impl<'a, A> Iterator for DListIterMut<'a, A>
+impl<'a, A> Iterator for DlistIterMut<'a, A>
 where
-    A: Adapter<Link = DListLink>,
+    A: Adapter<Link = DlistLink>,
 {
     type Item = &'a mut <A::Pointer as Pointer>::Item;
 
@@ -511,27 +511,27 @@ mod tests {
     use crate::intrusive_adapter;
 
     #[derive(Debug)]
-    struct DListItem {
-        link: DListLink,
+    struct DlistItem {
+        link: DlistLink,
         val: u64,
     }
 
-    impl DListItem {
+    impl DlistItem {
         fn new(val: u64) -> Self {
             Self {
-                link: DListLink::default(),
+                link: DlistLink::default(),
                 val,
             }
         }
     }
 
     #[derive(Debug, Default)]
-    struct DListAdapter;
+    struct DlistAdapter;
 
-    unsafe impl Adapter for DListAdapter {
-        type Pointer = Box<DListItem>;
+    unsafe impl Adapter for DlistAdapter {
+        type Pointer = Box<DlistItem>;
 
-        type Link = DListLink;
+        type Link = DlistLink;
 
         fn new() -> Self {
             Self
@@ -541,26 +541,26 @@ mod tests {
             &self,
             link: *const Self::Link,
         ) -> *const <Self::Pointer as Pointer>::Item {
-            crate::container_of!(link, DListItem, link)
+            crate::container_of!(link, DlistItem, link)
         }
 
         unsafe fn item2link(
             &self,
             item: *const <Self::Pointer as Pointer>::Item,
         ) -> *const Self::Link {
-            (item as *const u8).add(crate::offset_of!(DListItem, link)) as *const _
+            (item as *const u8).add(crate::offset_of!(DlistItem, link)) as *const _
         }
     }
 
-    intrusive_adapter! { DListArcAdapter = Arc<DListItem>: DListItem { link: DListLink } }
+    intrusive_adapter! { DlistArcAdapter = Arc<DlistItem>: DlistItem { link: DlistLink } }
 
     #[test]
     fn test_dlist_simple() {
-        let mut l = DList::<DListAdapter>::new();
+        let mut l = Dlist::<DlistAdapter>::new();
 
-        l.push_back(Box::new(DListItem::new(2)));
-        l.push_front(Box::new(DListItem::new(1)));
-        l.push_back(Box::new(DListItem::new(3)));
+        l.push_back(Box::new(DlistItem::new(2)));
+        l.push_front(Box::new(DlistItem::new(1)));
+        l.push_back(Box::new(DlistItem::new(3)));
 
         let v = l.iter_mut().map(|item| item.val).collect_vec();
         assert_eq!(v, vec![1, 2, 3]);
@@ -587,9 +587,9 @@ mod tests {
 
     #[test]
     fn test_arc_drop() {
-        let mut l = DList::<DListArcAdapter>::new();
+        let mut l = Dlist::<DlistArcAdapter>::new();
 
-        let items = (0..10).map(|i| Arc::new(DListItem::new(i))).collect_vec();
+        let items = (0..10).map(|i| Arc::new(DlistItem::new(i))).collect_vec();
         for item in items.iter() {
             l.push_back(item.clone());
         }
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index 49c2494b..16593a68 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -17,7 +17,7 @@ use std::{fmt::Debug, hash::Hasher, marker::PhantomData, ptr::NonNull};
 use foyer_common::code::{Key, Value};
 use twox_hash::XxHash64;
 
-use super::dlist::{DList, DListIter, DListIterMut, DListLink};
+use super::dlist::{Dlist, DlistIter, DlistIterMut, DlistLink};
 use crate::{
     core::{
         adapter::{Adapter, KeyAdapter, Link},
@@ -27,17 +27,17 @@ use crate::{
 };
 
 pub struct DuplicatedHashMapLink {
-    slot_link: DListLink,
-    group_link: DListLink,
-    group: DList<DuplicatedHashMapLinkGroupAdapter>,
+    slot_link: DlistLink,
+    group_link: DlistLink,
+    group: Dlist<DuplicatedHashMapLinkGroupAdapter>,
 }
 
 impl Default for DuplicatedHashMapLink {
     fn default() -> Self {
         Self {
-            slot_link: DListLink::default(),
-            group_link: DListLink::default(),
-            group: DList::new(),
+            slot_link: DlistLink::default(),
+            group_link: DlistLink::default(),
+            group: Dlist::new(),
         }
     }
 }
@@ -51,8 +51,8 @@ impl Debug for DuplicatedHashMapLink {
     }
 }
 
-intrusive_adapter! { DuplicatedHashMapLinkSlotAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { slot_link: DListLink } }
-intrusive_adapter! { DuplicatedHashMapLinkGroupAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { group_link: DListLink } }
+intrusive_adapter! { DuplicatedHashMapLinkSlotAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { slot_link: DlistLink } }
+intrusive_adapter! { DuplicatedHashMapLinkGroupAdapter = NonNull<DuplicatedHashMapLink>: DuplicatedHashMapLink { group_link: DlistLink } }
 
 impl DuplicatedHashMapLink {
     pub fn raw(&self) -> NonNull<DuplicatedHashMapLink> {
@@ -75,7 +75,7 @@ where
     V: Value,
     A: KeyAdapter<Key = K, Link = DuplicatedHashMapLink>,
 {
-    slots: Vec<DList<DuplicatedHashMapLinkSlotAdapter>>,
+    slots: Vec<Dlist<DuplicatedHashMapLinkSlotAdapter>>,
 
     len: usize,
 
@@ -119,7 +119,7 @@ where
     pub fn new(bits: usize) -> Self {
         let mut slots = Vec::with_capacity(1 << bits);
         for _ in 0..(1 << bits) {
-            slots.push(DList::new());
+            slots.push(Dlist::new());
         }
         Self {
             slots,
@@ -284,7 +284,7 @@ where
         &mut self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIterMut<'_, DuplicatedHashMapLinkSlotAdapter>> {
+    ) -> Option<DlistIterMut<'_, DuplicatedHashMapLinkSlotAdapter>> {
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
@@ -305,7 +305,7 @@ where
         &self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
+    ) -> Option<DlistIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 961d0566..8e715e17 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -17,7 +17,7 @@ use std::{hash::Hasher, marker::PhantomData, ptr::NonNull};
 use foyer_common::code::{Key, Value};
 use twox_hash::XxHash64;
 
-use super::dlist::{DList, DListIter, DListIterMut, DListLink};
+use super::dlist::{Dlist, DlistIter, DlistIterMut, DlistLink};
 use crate::{
     core::{
         adapter::{KeyAdapter, Link},
@@ -28,10 +28,10 @@ use crate::{
 
 #[derive(Debug, Default)]
 pub struct HashMapLink {
-    dlist_link: DListLink,
+    dlist_link: DlistLink,
 }
 
-intrusive_adapter! { HashMapLinkAdapter = NonNull<HashMapLink>: HashMapLink { dlist_link: DListLink } }
+intrusive_adapter! { HashMapLinkAdapter = NonNull<HashMapLink>: HashMapLink { dlist_link: DlistLink } }
 
 impl HashMapLink {
     pub fn raw(&self) -> NonNull<HashMapLink> {
@@ -54,7 +54,7 @@ where
     V: Value,
     A: KeyAdapter<Key = K, Link = HashMapLink>,
 {
-    slots: Vec<DList<HashMapLinkAdapter>>,
+    slots: Vec<Dlist<HashMapLinkAdapter>>,
 
     len: usize,
 
@@ -93,7 +93,7 @@ where
     pub fn new(bits: usize) -> Self {
         let mut slots = Vec::with_capacity(1 << bits);
         for _ in 0..(1 << bits) {
-            slots.push(DList::new());
+            slots.push(Dlist::new());
         }
         Self {
             slots,
@@ -189,7 +189,7 @@ where
         &mut self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIterMut<'_, HashMapLinkAdapter>> {
+    ) -> Option<DlistIterMut<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
@@ -210,7 +210,7 @@ where
         &self,
         key: &K,
         slot: usize,
-    ) -> Option<DListIter<'_, HashMapLinkAdapter>> {
+    ) -> Option<DlistIter<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
@@ -253,7 +253,7 @@ where
     A: KeyAdapter<Key = K, Link = HashMapLink>,
 {
     slot: usize,
-    iters: Vec<DListIter<'a, HashMapLinkAdapter>>,
+    iters: Vec<DlistIter<'a, HashMapLinkAdapter>>,
 
     adapter: A,
     _marker: PhantomData<(K, V)>,
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 32057323..4dcaf275 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -331,27 +331,27 @@ mod tests {
     use crate::{collections::dlist::*, intrusive_adapter};
 
     #[derive(Debug)]
-    struct DListItem {
-        link: DListLink,
+    struct DlistItem {
+        link: DlistLink,
         val: u64,
     }
 
-    impl DListItem {
+    impl DlistItem {
         fn new(val: u64) -> Self {
             Self {
-                link: DListLink::default(),
+                link: DlistLink::default(),
                 val,
             }
         }
     }
 
-    intrusive_adapter! { DListItemAdapter = Box<DListItem>: DListItem { link: DListLink }}
-    key_adapter! { DListItemAdapter = DListItem { val: u64 } }
+    intrusive_adapter! { DlistItemAdapter = Box<DlistItem>: DlistItem { link: DlistLink }}
+    key_adapter! { DlistItemAdapter = DlistItem { val: u64 } }
 
     #[test]
     fn test_adapter_macro() {
-        let mut l = DList::<DListItemAdapter>::new();
-        l.push_front(Box::new(DListItem::new(1)));
+        let mut l = Dlist::<DlistItemAdapter>::new();
+        l.push_front(Box::new(DlistItem::new(1)));
         let v = l.iter().map(|item| item.val).collect_vec();
         assert_eq!(v, vec![1]);
     }
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 494b9b4e..2a97c5aa 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -30,7 +30,7 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, Link},
         pointer::Pointer,
@@ -43,7 +43,7 @@ pub struct FifoConfig;
 
 #[derive(Debug, Default)]
 pub struct FifoLink {
-    link: DListLink,
+    link: DlistLink,
 }
 
 impl FifoLink {
@@ -58,7 +58,7 @@ impl Link for FifoLink {
     }
 }
 
-intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DListLink } }
+intrusive_adapter! { FifoLinkAdapter = NonNull<FifoLink>: FifoLink { link: DlistLink } }
 
 /// FIFO policy
 #[derive(Debug)]
@@ -67,7 +67,7 @@ where
     A: Adapter<Link = FifoLink>,
     <A as Adapter>::Pointer: Clone,
 {
-    queue: DList<FifoLinkAdapter>,
+    queue: Dlist<FifoLinkAdapter>,
 
     _config: FifoConfig,
 
@@ -99,7 +99,7 @@ where
 {
     pub fn new(config: FifoConfig) -> Self {
         Self {
-            queue: DList::new(),
+            queue: Dlist::new(),
 
             _config: config,
 
@@ -165,7 +165,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     fifo: &'a Fifo<A>,
-    iter: DListIter<'a, FifoLinkAdapter>,
+    iter: DlistIter<'a, FifoLinkAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 6893fab2..fbb01684 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -37,7 +37,7 @@ use twox_hash::XxHash64;
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, KeyAdapter, Link},
         pointer::Pointer,
@@ -69,8 +69,8 @@ enum LruType {
 
 #[derive(Debug, Default)]
 pub struct LfuLink {
-    link_tiny: DListLink,
-    link_main: DListLink,
+    link_tiny: DlistLink,
+    link_main: DlistLink,
 }
 
 impl Link for LfuLink {
@@ -94,8 +94,8 @@ impl LfuLink {
     }
 }
 
-intrusive_adapter! { LfuLinkTinyDListAdapter = NonNull<LfuLink>: LfuLink { link_tiny: DListLink } }
-intrusive_adapter! { LfuLinkMainDListAdapter = NonNull<LfuLink>: LfuLink { link_main: DListLink } }
+intrusive_adapter! { LfuLinkTinyDlistAdapter = NonNull<LfuLink>: LfuLink { link_tiny: DlistLink } }
+intrusive_adapter! { LfuLinkMainDlistAdapter = NonNull<LfuLink>: LfuLink { link_main: DlistLink } }
 
 /// Implements the W-TinyLFU cache eviction policy as described in -
 ///
@@ -131,10 +131,10 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     /// tiny lru list
-    lru_tiny: DList<LfuLinkTinyDListAdapter>,
+    lru_tiny: Dlist<LfuLinkTinyDlistAdapter>,
 
     /// main lru list
-    lru_main: DList<LfuLinkMainDListAdapter>,
+    lru_main: Dlist<LfuLinkMainDlistAdapter>,
 
     /// the window length counter
     window_size: usize,
@@ -180,8 +180,8 @@ where
 {
     pub fn new(config: LfuConfig) -> Self {
         let mut res = Self {
-            lru_tiny: DList::new(),
-            lru_main: DList::new(),
+            lru_tiny: Dlist::new(),
+            lru_main: Dlist::new(),
 
             window_size: 0,
             max_window_size: 0,
@@ -415,8 +415,8 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     lfu: &'a Lfu<A>,
-    iter_tiny: DListIter<'a, LfuLinkTinyDListAdapter>,
-    iter_main: DListIter<'a, LfuLinkMainDListAdapter>,
+    iter_tiny: DlistIter<'a, LfuLinkTinyDlistAdapter>,
+    iter_main: DlistIter<'a, LfuLinkMainDlistAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 2915b0e5..dba2e8de 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -30,7 +30,7 @@ use std::{mem::ManuallyDrop, ptr::NonNull};
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, Link},
         pointer::Pointer,
@@ -46,7 +46,7 @@ pub struct LruConfig {
 
 #[derive(Debug, Default)]
 pub struct LruLink {
-    link_lru: DListLink,
+    link_lru: DlistLink,
 
     is_in_tail: bool,
 }
@@ -63,7 +63,7 @@ impl Link for LruLink {
     }
 }
 
-intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DListLink } }
+intrusive_adapter! { LruLinkAdapter = NonNull<LruLink>: LruLink { link_lru: DlistLink } }
 
 #[derive(Debug)]
 pub struct Lru<A>
@@ -72,7 +72,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     /// lru list
-    lru: DList<LruLinkAdapter>,
+    lru: Dlist<LruLinkAdapter>,
 
     /// insertion point
     insertion_point: Option<NonNull<LruLink>>,
@@ -110,7 +110,7 @@ where
 {
     fn new(config: LruConfig) -> Self {
         Self {
-            lru: DList::new(),
+            lru: Dlist::new(),
 
             insertion_point: None,
 
@@ -146,7 +146,7 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            self.ensuer_not_insertion_point(link);
+            self.ensure_not_insertion_point(link);
             self.lru
                 .iter_mut_from_raw(link.as_ref().link_lru.raw())
                 .remove()
@@ -169,7 +169,7 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            self.ensuer_not_insertion_point(link);
+            self.ensure_not_insertion_point(link);
 
             self.move_to_lru_front(link);
 
@@ -239,7 +239,7 @@ where
         }
     }
 
-    unsafe fn ensuer_not_insertion_point(&mut self, link: NonNull<LruLink>) {
+    unsafe fn ensure_not_insertion_point(&mut self, link: NonNull<LruLink>) {
         if Some(link) == self.insertion_point {
             self.insertion_point = self.lru_prev(link);
             match &mut self.insertion_point {
@@ -290,7 +290,7 @@ where
     <A as Adapter>::Pointer: Clone,
 {
     lru: &'a Lru<A>,
-    iter: DListIter<'a, LruLinkAdapter>,
+    iter: DlistIter<'a, LruLinkAdapter>,
 
     ptr: ManuallyDrop<Option<<A as Adapter>::Pointer>>,
 }
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index a7dca2ad..2fea2bad 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -32,7 +32,7 @@ use itertools::Itertools;
 
 use super::EvictionPolicy;
 use crate::{
-    collections::dlist::{DList, DListIter, DListLink},
+    collections::dlist::{Dlist, DlistIter, DlistLink},
     core::{
         adapter::{Adapter, Link, PriorityAdapter},
         pointer::Pointer,
@@ -52,7 +52,7 @@ pub struct SegmentedFifoConfig {
 
 #[derive(Debug, Default)]
 pub struct SegmentedFifoLink {
-    link_queue: DListLink,
+    link_queue: DlistLink,
 
     priority: usize,
 }
@@ -69,7 +69,7 @@ impl Link for SegmentedFifoLink {
     }
 }
 
-intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: SegmentedFifoLink { link_queue: DListLink } }
+intrusive_adapter! { SegmentedFifoLinkAdapter = NonNull<SegmentedFifoLink>: SegmentedFifoLink { link_queue: DlistLink } }
 
 /// Segmented FIFO policy
 ///
@@ -109,7 +109,7 @@ where
     A::Pointer: Clone,
 {
     // Note: All queue share the same dlist link.
-    segments: Vec<DList<SegmentedFifoLinkAdapter>>,
+    segments: Vec<Dlist<SegmentedFifoLinkAdapter>>,
 
     config: SegmentedFifoConfig,
     total_ratio: usize,
@@ -142,7 +142,7 @@ where
 {
     pub fn new(config: SegmentedFifoConfig) -> Self {
         let segments = (0..config.segment_ratios.len())
-            .map(|_| DList::new())
+            .map(|_| Dlist::new())
             .collect_vec();
         let total_ratio = config.segment_ratios.iter().sum();
 
@@ -249,7 +249,7 @@ where
     A::Pointer: Clone,
 {
     sfifo: &'a SegmentedFifo<A>,
-    iter_segments: Vec<DListIter<'a, SegmentedFifoLinkAdapter>>,
+    iter_segments: Vec<DlistIter<'a, SegmentedFifoLinkAdapter>>,
     segment: usize,
 
     ptr: ManuallyDrop<Option<A::Pointer>>,

From 80755b78aa35c10bbe4ff0069f585d5a59eba228 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 29 Feb 2024 14:31:58 +0800
Subject: [PATCH 213/261] refactor: use NonNull instead of raw pointer in
 foyer-intrusive (#274)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/collections/dlist.rs      | 69 ++++++++++---------
 .../src/collections/duplicated_hashmap.rs     | 33 +++++----
 foyer-intrusive/src/collections/hashmap.rs    | 36 +++++-----
 foyer-intrusive/src/core/adapter.rs           | 43 +++++++-----
 foyer-intrusive/src/eviction/fifo.rs          | 14 ++--
 foyer-intrusive/src/eviction/lfu.rs           | 22 +++---
 foyer-intrusive/src/eviction/lru.rs           | 18 ++---
 foyer-intrusive/src/eviction/sfifo.rs         | 18 +++--
 8 files changed, 133 insertions(+), 120 deletions(-)

diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index 93ed9b98..839a88ef 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -93,34 +93,32 @@ where
     pub fn front(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         unsafe {
             self.head
-                .map(|link| self.adapter.link2item(link.as_ptr()))
-                .map(|link| &*link)
+                .map(|link| self.adapter.link2item(link))
+                .map(|link| link.as_ref())
         }
     }
 
     pub fn back(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         unsafe {
             self.tail
-                .map(|link| self.adapter.link2item(link.as_ptr()))
-                .map(|link| &*link)
+                .map(|link| self.adapter.link2item(link))
+                .map(|link| link.as_ref())
         }
     }
 
     pub fn front_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         unsafe {
             self.head
-                .map(|link| self.adapter.link2item(link.as_ptr()))
-                .map(|link| link as *mut _)
-                .map(|link| &mut *link)
+                .map(|link| self.adapter.link2item(link))
+                .map(|mut link| link.as_mut())
         }
     }
 
     pub fn back_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         unsafe {
             self.tail
-                .map(|link| self.adapter.link2item(link.as_ptr()))
-                .map(|link| link as *mut _)
-                .map(|link| &mut *link)
+                .map(|link| self.adapter.link2item(link))
+                .map(|mut link| link.as_mut())
         }
     }
 
@@ -166,6 +164,17 @@ where
         self.len() == 0
     }
 
+    /// Remove an node that holds the given raw link.
+    ///
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn remove_raw(&mut self, link: NonNull<DlistLink>) -> A::Pointer {
+        let mut iter = self.iter_mut_from_raw(link);
+        debug_assert!(iter.is_valid());
+        iter.remove().unwrap_unchecked()
+    }
+
     /// Create mutable iterator directly on raw link.
     ///
     /// # Safety
@@ -226,7 +235,7 @@ where
 
     pub fn get(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         self.link
-            .map(|link| unsafe { &*self.dlist.adapter.link2item(link.as_ptr()) })
+            .map(|link| unsafe { self.dlist.adapter.link2item(link).as_ref() })
     }
 
     /// Move to next.
@@ -292,12 +301,12 @@ where
 
     pub fn get(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         self.link
-            .map(|link| unsafe { &*self.dlist.adapter.link2item(link.as_ptr()) })
+            .map(|link| unsafe { self.dlist.adapter.link2item(link).as_ref() })
     }
 
     pub fn get_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         self.link
-            .map(|link| unsafe { &mut *(self.dlist.adapter.link2item(link.as_ptr()) as *mut _) })
+            .map(|link| unsafe { self.dlist.adapter.link2item(link).as_mut() })
     }
 
     /// Move to next.
@@ -345,8 +354,8 @@ where
 
             let mut link = self.link.unwrap();
 
-            let item = self.dlist.adapter.link2item(link.as_ptr());
-            let ptr = A::Pointer::from_raw(item);
+            let item = self.dlist.adapter.link2item(link);
+            let ptr = A::Pointer::from_raw(item.as_ptr());
 
             // fix head and tail if node is either of that
             let mut prev = link.as_ref().prev;
@@ -383,9 +392,8 @@ where
     /// If iter is on null, link to tail.
     pub fn insert_before(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = A::Pointer::into_raw(ptr);
-            let mut link_new =
-                NonNull::new_unchecked(self.dlist.adapter.item2link(item_new) as *mut A::Link);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let mut link_new = self.dlist.adapter.item2link(item_new);
             assert!(!link_new.as_ref().is_linked());
 
             match self.link {
@@ -411,9 +419,8 @@ where
     /// If iter is on null, link to head.
     pub fn insert_after(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = A::Pointer::into_raw(ptr);
-            let mut link_new =
-                NonNull::new_unchecked(self.dlist.adapter.item2link(item_new) as *mut A::Link);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let mut link_new = self.dlist.adapter.item2link(item_new);
             assert!(!link_new.as_ref().is_linked());
 
             match self.link {
@@ -476,7 +483,7 @@ where
     fn next(&mut self) -> Option<Self::Item> {
         self.next();
         match self.link {
-            Some(link) => Some(unsafe { &*(self.dlist.adapter.link2item(link.as_ptr())) }),
+            Some(link) => Some(unsafe { self.dlist.adapter.link2item(link).as_ref() }),
             None => None,
         }
     }
@@ -491,9 +498,7 @@ where
     fn next(&mut self) -> Option<Self::Item> {
         self.next();
         match self.link {
-            Some(link) => {
-                Some(unsafe { &mut *(self.dlist.adapter.link2item(link.as_ptr()) as *mut _) })
-            }
+            Some(link) => Some(unsafe { self.dlist.adapter.link2item(link).as_mut() }),
             None => None,
         }
     }
@@ -539,16 +544,18 @@ mod tests {
 
         unsafe fn link2item(
             &self,
-            link: *const Self::Link,
-        ) -> *const <Self::Pointer as Pointer>::Item {
-            crate::container_of!(link, DlistItem, link)
+            link: NonNull<Self::Link>,
+        ) -> NonNull<<Self::Pointer as Pointer>::Item> {
+            NonNull::new_unchecked(crate::container_of!(link.as_ptr(), DlistItem, link) as *mut _)
         }
 
         unsafe fn item2link(
             &self,
-            item: *const <Self::Pointer as Pointer>::Item,
-        ) -> *const Self::Link {
-            (item as *const u8).add(crate::offset_of!(DlistItem, link)) as *const _
+            item: NonNull<<Self::Pointer as Pointer>::Item>,
+        ) -> NonNull<Self::Link> {
+            NonNull::new_unchecked(
+                (item.as_ptr() as *const u8).add(crate::offset_of!(DlistItem, link)) as *mut _,
+            )
         }
     }
 
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index 16593a68..7f902131 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -101,8 +101,8 @@ where
                     iter_group.front();
                     while iter_group.is_valid() {
                         let link_group = iter_group.remove().unwrap();
-                        let item = self.adapter.link2item(link_group.as_ptr());
-                        let _ = A::Pointer::from_raw(item);
+                        let item = self.adapter.link2item(link_group);
+                        let _ = A::Pointer::from_raw(item.as_ptr());
                     }
                 }
             }
@@ -131,13 +131,12 @@ where
 
     pub fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = A::Pointer::into_raw(ptr);
-            let mut link_new =
-                NonNull::new_unchecked(self.adapter.item2link(item_new) as *mut A::Link);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let mut link_new = self.adapter.item2link(item_new);
 
             assert!(link_new.as_ref().group.is_empty());
 
-            let key_new = &*self.adapter.item2key(item_new);
+            let key_new = self.adapter.item2key(item_new).as_ref();
             let hash = self.hash_key(key_new);
             let slot = (self.slots.len() - 1) & hash as usize;
 
@@ -170,8 +169,8 @@ where
                     while iter.is_valid() {
                         let link = iter.remove().unwrap();
                         debug_assert!(!link.as_ref().is_linked());
-                        let item = self.adapter.link2item(link.as_ptr());
-                        let ptr = A::Pointer::from_raw(item);
+                        let item = self.adapter.link2item(link);
+                        let ptr = A::Pointer::from_raw(item.as_ptr());
                         res.push(ptr);
                     }
                     debug_assert!(link.as_ref().group.is_empty());
@@ -196,7 +195,7 @@ where
                     iter.front();
                     while iter.is_valid() {
                         let link = iter.get().unwrap();
-                        let item = &*self.adapter.link2item(link as *const _);
+                        let item = self.adapter.link2item(link.raw()).as_ref();
                         res.push(item);
                         iter.next();
                     }
@@ -223,8 +222,8 @@ where
         mut link: NonNull<DuplicatedHashMapLink>,
     ) -> A::Pointer {
         assert!(link.as_ref().is_linked());
-        let item = self.adapter.link2item(link.as_ptr());
-        let key = &*self.adapter.item2key(item);
+        let item = self.adapter.link2item(link);
+        let key = self.adapter.item2key(item).as_ref();
         let hash = self.hash_key(key);
         let slot = (self.slots.len() - 1) & hash as usize;
 
@@ -257,7 +256,7 @@ where
                     header = &*header.prev().unwrap().as_ptr();
                 }
                 let adapter = DuplicatedHashMapLinkGroupAdapter::new();
-                &mut *(adapter.link2item(header as *const _) as *mut DuplicatedHashMapLink)
+                adapter.link2item(header.raw()).as_mut()
             };
             debug_assert!(header_link.slot_link.is_linked());
 
@@ -274,7 +273,7 @@ where
         debug_assert!(!link.as_ref().group_link.is_linked());
         debug_assert!(link.as_ref().group.is_empty());
 
-        A::Pointer::from_raw(item)
+        A::Pointer::from_raw(item.as_ptr())
     }
 
     /// # Safety
@@ -288,8 +287,8 @@ where
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
-            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
-            let ikey = &*self.adapter.item2key(item);
+            let item = self.adapter.link2item(iter.get().unwrap().raw());
+            let ikey = self.adapter.item2key(item).as_ref();
             if ikey == key {
                 return Some(iter);
             }
@@ -309,8 +308,8 @@ where
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
-            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
-            let ikey = &*self.adapter.item2key(item);
+            let item = self.adapter.link2item(iter.get().unwrap().raw());
+            let ikey = self.adapter.item2key(item).as_ref();
             if ikey == key {
                 return Some(iter);
             }
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 8e715e17..45d9762c 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -76,8 +76,8 @@ where
                 iter.front();
                 while iter.is_valid() {
                     let link = iter.remove().unwrap();
-                    let item = self.adapter.link2item(link.as_ptr());
-                    let _ = A::Pointer::from_raw(item);
+                    let item = self.adapter.link2item(link);
+                    let _ = A::Pointer::from_raw(item.as_ptr());
                 }
             }
         }
@@ -105,10 +105,10 @@ where
 
     pub fn insert(&mut self, ptr: A::Pointer) -> Option<A::Pointer> {
         unsafe {
-            let item_new = A::Pointer::into_raw(ptr);
-            let link_new = NonNull::new_unchecked(self.adapter.item2link(item_new) as *mut A::Link);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let link_new = self.adapter.item2link(item_new);
 
-            let key_new = &*self.adapter.item2key(item_new);
+            let key_new = self.adapter.item2key(item_new).as_ref();
             let hash = self.hash_key(key_new);
             let slot = (self.slots.len() - 1) & hash as usize;
 
@@ -145,8 +145,8 @@ where
 
             match self.lookup_inner(key, slot) {
                 Some(iter) => {
-                    let link = iter.get().unwrap() as *const _;
-                    let item = &*self.adapter.link2item(link);
+                    let link = iter.get().unwrap().raw();
+                    let item = self.adapter.link2item(link).as_ref();
                     Some(item)
                 }
                 None => None,
@@ -171,15 +171,15 @@ where
     /// `link` MUST be in this [`HashMap`].
     pub unsafe fn remove_in_place(&mut self, link: NonNull<HashMapLink>) -> A::Pointer {
         assert!(link.as_ref().is_linked());
-        let item = self.adapter.link2item(link.as_ptr());
-        let key = &*self.adapter.item2key(item);
+        let item = self.adapter.link2item(link);
+        let key = self.adapter.item2key(item).as_ref();
         let hash = self.hash_key(key);
         let slot = (self.slots.len() - 1) & hash as usize;
         self.slots[slot]
             .iter_mut_from_raw(link.as_ref().dlist_link.raw())
             .remove();
         self.len -= 1;
-        A::Pointer::from_raw(item)
+        A::Pointer::from_raw(item.as_ptr())
     }
 
     /// # Safety
@@ -193,8 +193,8 @@ where
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
-            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
-            let ikey = &*self.adapter.item2key(item);
+            let item = self.adapter.link2item(iter.get().unwrap().raw());
+            let ikey = self.adapter.item2key(item).as_ref();
             if ikey == key {
                 return Some(iter);
             }
@@ -214,8 +214,8 @@ where
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
-            let item = self.adapter.link2item(iter.get().unwrap().raw().as_ptr());
-            let ikey = &*self.adapter.item2key(item);
+            let item = self.adapter.link2item(iter.get().unwrap().raw());
+            let ikey = self.adapter.item2key(item).as_ref();
             if ikey == key {
                 return Some(iter);
             }
@@ -231,8 +231,8 @@ where
         match self.lookup_inner_mut(key, slot) {
             Some(mut iter) => {
                 let link = iter.remove().unwrap();
-                let item = self.adapter.link2item(link.as_ptr());
-                let ptr = A::Pointer::from_raw(item);
+                let item = self.adapter.link2item(link);
+                let ptr = A::Pointer::from_raw(item.as_ptr());
                 Some(ptr)
             }
             None => None,
@@ -287,13 +287,13 @@ where
     pub fn get(&self) -> Option<&<A::Pointer as Pointer>::Item> {
         self.iters[self.slot]
             .get()
-            .map(|link| unsafe { &*(self.adapter.link2item(link.raw().as_ptr()) as *const _) })
+            .map(|link| unsafe { self.adapter.link2item(link.raw()).as_ref() })
     }
 
     pub fn get_mut(&mut self) -> Option<&mut <A::Pointer as Pointer>::Item> {
         self.iters[self.slot]
             .get()
-            .map(|link| unsafe { &mut *(self.adapter.link2item(link.raw().as_ptr()) as *mut _) })
+            .map(|link| unsafe { self.adapter.link2item(link.raw()).as_mut() })
     }
 
     /// Move to next.
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 4dcaf275..f8a4946a 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -50,12 +50,18 @@ pub unsafe trait Adapter: Send + Sync + Debug + 'static {
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn link2item(&self, link: *const Self::Link) -> *const <Self::Pointer as Pointer>::Item;
+    unsafe fn link2item(
+        &self,
+        link: std::ptr::NonNull<Self::Link>,
+    ) -> std::ptr::NonNull<<Self::Pointer as Pointer>::Item>;
 
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn item2link(&self, item: *const <Self::Pointer as Pointer>::Item) -> *const Self::Link;
+    unsafe fn item2link(
+        &self,
+        item: std::ptr::NonNull<<Self::Pointer as Pointer>::Item>,
+    ) -> std::ptr::NonNull<Self::Link>;
 }
 
 /// # Safety
@@ -69,7 +75,10 @@ pub unsafe trait KeyAdapter: Adapter {
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn item2key(&self, item: *const <Self::Pointer as Pointer>::Item) -> *const Self::Key;
+    unsafe fn item2key(
+        &self,
+        item: std::ptr::NonNull<<Self::Pointer as Pointer>::Item>,
+    ) -> std::ptr::NonNull<Self::Key>;
 }
 
 /// # Safety
@@ -85,8 +94,8 @@ pub unsafe trait PriorityAdapter: Adapter {
     /// Pointer operations MUST be valid.
     unsafe fn item2priority(
         &self,
-        item: *const <Self::Pointer as Pointer>::Item,
-    ) -> *const Self::Priority;
+        item: std::ptr::NonNull<<Self::Pointer as Pointer>::Item>,
+    ) -> std::ptr::NonNull<Self::Priority>;
 }
 
 /// Macro to generate an implementation of [`Adapter`] for instrusive container and items.
@@ -150,16 +159,16 @@ macro_rules! intrusive_adapter {
 
             unsafe fn link2item(
                 &self,
-                link: *const Self::Link,
-            ) -> *const <Self::Pointer as $crate::core::pointer::Pointer>::Item {
-                $crate::container_of!(link, $item, $field)
+                link: std::ptr::NonNull<Self::Link>,
+            ) -> std::ptr::NonNull<<Self::Pointer as $crate::core::pointer::Pointer>::Item> {
+                std::ptr::NonNull::new_unchecked($crate::container_of!(link.as_ptr(), $item, $field) as *mut _)
             }
 
             unsafe fn item2link(
                 &self,
-                item: *const <Self::Pointer as $crate::core::pointer::Pointer>::Item,
-            ) -> *const Self::Link {
-                (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
+                item: std::ptr::NonNull<<Self::Pointer as $crate::core::pointer::Pointer>::Item>,
+            ) -> std::ptr::NonNull<Self::Link> {
+                std::ptr::NonNull::new_unchecked((item.as_ptr() as *mut u8).add($crate::offset_of!($item, $field)) as *mut Self::Link)
             }
         }
 
@@ -232,9 +241,9 @@ macro_rules! key_adapter {
 
             unsafe fn item2key(
                 &self,
-                item: *const <Self::Pointer as $crate::core::pointer::Pointer>::Item,
-            ) -> *const Self::Key {
-                (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
+                item: std::ptr::NonNull<<Self::Pointer as $crate::core::pointer::Pointer>::Item>,
+            ) -> std::ptr::NonNull<Self::Key> {
+                std::ptr::NonNull::new_unchecked((item.as_ptr() as *const u8).add($crate::offset_of!($item, $field)) as *mut _)
             }
         }
     };
@@ -301,9 +310,9 @@ macro_rules! priority_adapter {
 
             unsafe fn item2priority(
                 &self,
-                item: *const <Self::Pointer as $crate::core::pointer::Pointer>::Item,
-            ) -> *const Self::Priority {
-                (item as *const u8).add($crate::offset_of!($item, $field)) as *const _
+                item: std::ptr::NonNull< <Self::Pointer as $crate::core::pointer::Pointer>::Item>,
+            ) -> std::ptr::NonNull<Self::Priority>{
+                std::ptr::NonNull::new_unchecked((item.as_ptr() as *const u8).add($crate::offset_of!($item, $field)) as *mut _)
             }
         }
     };
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 2a97c5aa..5286f77a 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -111,8 +111,8 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = A::Pointer::into_raw(ptr);
-            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
+            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
 
@@ -124,8 +124,8 @@ where
 
     fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = A::Pointer::as_ptr(ptr);
-            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut FifoLink);
+            let item = NonNull::new_unchecked(A::Pointer::as_ptr(ptr) as *mut _);
+            let link = self.adapter.item2link(item);
 
             assert!(link.as_ref().is_linked());
 
@@ -136,7 +136,7 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item)
+            A::Pointer::from_raw(item.as_ptr())
         }
     }
 
@@ -178,8 +178,8 @@ where
     unsafe fn update_ptr(&mut self, link: NonNull<FifoLink>) {
         std::mem::forget(self.ptr.take());
 
-        let item = self.fifo.adapter.link2item(link.as_ptr());
-        let ptr = A::Pointer::from_raw(item);
+        let item = self.fifo.adapter.link2item(link);
+        let ptr = A::Pointer::from_raw(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index fbb01684..a7b7a9e7 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -202,8 +202,8 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = A::Pointer::into_raw(ptr);
-            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
+            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
 
@@ -232,8 +232,8 @@ where
 
     fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = A::Pointer::as_ptr(ptr);
-            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
+            let item = NonNull::new_unchecked(A::Pointer::as_ptr(ptr) as *mut _);
+            let link = self.adapter.item2link(item);
 
             assert!(link.as_ref().is_linked());
 
@@ -241,14 +241,14 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item)
+            A::Pointer::from_raw(item.as_ptr())
         }
     }
 
     fn access(&mut self, ptr: &A::Pointer) {
         unsafe {
-            let item = A::Pointer::as_ptr(ptr);
-            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LfuLink);
+            let item = NonNull::new_unchecked(A::Pointer::as_ptr(ptr) as *mut _);
+            let link = self.adapter.item2link(item);
 
             assert!(link.as_ref().is_linked());
 
@@ -400,9 +400,9 @@ where
     fn hash_link(&self, link: NonNull<LfuLink>) -> u64 {
         let mut hasher = XxHash64::default();
         let key = unsafe {
-            let item = self.adapter.link2item(link.as_ptr());
+            let item = self.adapter.link2item(link);
             let key = self.adapter.item2key(item);
-            &*key
+            key.as_ref()
         };
         key.hash(&mut hasher);
         hasher.finish()
@@ -429,8 +429,8 @@ where
     unsafe fn update_ptr(&mut self, link: NonNull<LfuLink>) {
         std::mem::forget(self.ptr.take());
 
-        let item = self.lfu.adapter.link2item(link.as_ptr());
-        let ptr = A::Pointer::from_raw(item);
+        let item = self.lfu.adapter.link2item(link);
+        let ptr = A::Pointer::from_raw(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index dba2e8de..a602bb16 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -126,8 +126,8 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = A::Pointer::into_raw(ptr);
-            let link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
+            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
 
@@ -141,8 +141,8 @@ where
 
     fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = A::Pointer::as_ptr(ptr);
-            let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
+            let item = NonNull::new_unchecked(A::Pointer::as_ptr(ptr) as *mut _);
+            let mut link = self.adapter.item2link(item);
 
             assert!(link.as_ref().is_linked());
 
@@ -158,14 +158,14 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item)
+            A::Pointer::from_raw(item.as_ptr())
         }
     }
 
     fn access(&mut self, ptr: &A::Pointer) {
         unsafe {
-            let item = A::Pointer::as_ptr(ptr);
-            let mut link = NonNull::new_unchecked(self.adapter.item2link(item) as *mut LruLink);
+            let item = NonNull::new_unchecked(A::Pointer::as_ptr(ptr) as *mut _);
+            let mut link = self.adapter.item2link(item);
 
             assert!(link.as_ref().is_linked());
 
@@ -303,8 +303,8 @@ where
     unsafe fn update_ptr(&mut self, link: NonNull<LruLink>) {
         std::mem::forget(self.ptr.take());
 
-        let item = self.lru.adapter.link2item(link.as_ptr());
-        let ptr = A::Pointer::from_raw(item);
+        let item = self.lru.adapter.link2item(link);
+        let ptr = A::Pointer::from_raw(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index 2fea2bad..5c057c24 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -160,13 +160,12 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = A::Pointer::into_raw(ptr);
-            let mut link =
-                NonNull::new_unchecked(self.adapter.item2link(item) as *mut SegmentedFifoLink);
+            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let mut link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
 
-            let priority = *self.adapter.item2priority(item);
+            let priority = *self.adapter.item2priority(item).as_ref();
             link.as_mut().priority = priority.into();
 
             self.segments[priority.into()].push_back(link);
@@ -179,9 +178,8 @@ where
 
     fn remove(&mut self, ptr: &A::Pointer) -> A::Pointer {
         unsafe {
-            let item = A::Pointer::as_ptr(ptr);
-            let link =
-                NonNull::new_unchecked(self.adapter.item2link(item) as *mut SegmentedFifoLink);
+            let item = NonNull::new_unchecked(A::Pointer::as_ptr(ptr) as *mut _);
+            let link = self.adapter.item2link(item);
 
             assert!(link.as_ref().is_linked());
 
@@ -195,7 +193,7 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item)
+            A::Pointer::from_raw(item.as_ptr())
         }
     }
 
@@ -263,8 +261,8 @@ where
     unsafe fn update_ptr(&mut self, link: NonNull<SegmentedFifoLink>) {
         std::mem::forget(self.ptr.take());
 
-        let item = self.sfifo.adapter.link2item(link.as_ptr());
-        let ptr = A::Pointer::from_raw(item);
+        let item = self.sfifo.adapter.link2item(link);
+        let ptr = A::Pointer::from_raw(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 

From 83d567810fe8a24a254decb795e6a07f57c965e7 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 29 Feb 2024 14:47:41 +0800
Subject: [PATCH 214/261] chore: rename fn name for Pointer (#275)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/collections/dlist.rs      |   6 +-
 .../src/collections/duplicated_hashmap.rs     |   8 +-
 foyer-intrusive/src/collections/hashmap.rs    |   8 +-
 foyer-intrusive/src/core/pointer.rs           | 100 +++++++++---------
 foyer-intrusive/src/eviction/fifo.rs          |   6 +-
 foyer-intrusive/src/eviction/lfu.rs           |   6 +-
 foyer-intrusive/src/eviction/lru.rs           |   6 +-
 foyer-intrusive/src/eviction/sfifo.rs         |   6 +-
 8 files changed, 73 insertions(+), 73 deletions(-)

diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index 839a88ef..c3c6124b 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -355,7 +355,7 @@ where
             let mut link = self.link.unwrap();
 
             let item = self.dlist.adapter.link2item(link);
-            let ptr = A::Pointer::from_raw(item.as_ptr());
+            let ptr = A::Pointer::from_ptr(item.as_ptr());
 
             // fix head and tail if node is either of that
             let mut prev = link.as_ref().prev;
@@ -392,7 +392,7 @@ where
     /// If iter is on null, link to tail.
     pub fn insert_before(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let mut link_new = self.dlist.adapter.item2link(item_new);
             assert!(!link_new.as_ref().is_linked());
 
@@ -419,7 +419,7 @@ where
     /// If iter is on null, link to head.
     pub fn insert_after(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let mut link_new = self.dlist.adapter.item2link(item_new);
             assert!(!link_new.as_ref().is_linked());
 
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index 7f902131..ec803f44 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -102,7 +102,7 @@ where
                     while iter_group.is_valid() {
                         let link_group = iter_group.remove().unwrap();
                         let item = self.adapter.link2item(link_group);
-                        let _ = A::Pointer::from_raw(item.as_ptr());
+                        let _ = A::Pointer::from_ptr(item.as_ptr());
                     }
                 }
             }
@@ -131,7 +131,7 @@ where
 
     pub fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let mut link_new = self.adapter.item2link(item_new);
 
             assert!(link_new.as_ref().group.is_empty());
@@ -170,7 +170,7 @@ where
                         let link = iter.remove().unwrap();
                         debug_assert!(!link.as_ref().is_linked());
                         let item = self.adapter.link2item(link);
-                        let ptr = A::Pointer::from_raw(item.as_ptr());
+                        let ptr = A::Pointer::from_ptr(item.as_ptr());
                         res.push(ptr);
                     }
                     debug_assert!(link.as_ref().group.is_empty());
@@ -273,7 +273,7 @@ where
         debug_assert!(!link.as_ref().group_link.is_linked());
         debug_assert!(link.as_ref().group.is_empty());
 
-        A::Pointer::from_raw(item.as_ptr())
+        A::Pointer::from_ptr(item.as_ptr())
     }
 
     /// # Safety
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 45d9762c..3e1f1969 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -77,7 +77,7 @@ where
                 while iter.is_valid() {
                     let link = iter.remove().unwrap();
                     let item = self.adapter.link2item(link);
-                    let _ = A::Pointer::from_raw(item.as_ptr());
+                    let _ = A::Pointer::from_ptr(item.as_ptr());
                 }
             }
         }
@@ -105,7 +105,7 @@ where
 
     pub fn insert(&mut self, ptr: A::Pointer) -> Option<A::Pointer> {
         unsafe {
-            let item_new = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item_new = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let link_new = self.adapter.item2link(item_new);
 
             let key_new = self.adapter.item2key(item_new).as_ref();
@@ -179,7 +179,7 @@ where
             .iter_mut_from_raw(link.as_ref().dlist_link.raw())
             .remove();
         self.len -= 1;
-        A::Pointer::from_raw(item.as_ptr())
+        A::Pointer::from_ptr(item.as_ptr())
     }
 
     /// # Safety
@@ -232,7 +232,7 @@ where
             Some(mut iter) => {
                 let link = iter.remove().unwrap();
                 let item = self.adapter.link2item(link);
-                let ptr = A::Pointer::from_raw(item.as_ptr());
+                let ptr = A::Pointer::from_ptr(item.as_ptr());
                 Some(ptr)
             }
             None => None,
diff --git a/foyer-intrusive/src/core/pointer.rs b/foyer-intrusive/src/core/pointer.rs
index 53c6dcc8..8c74d48d 100644
--- a/foyer-intrusive/src/core/pointer.rs
+++ b/foyer-intrusive/src/core/pointer.rs
@@ -37,9 +37,9 @@ pub unsafe trait Pointer {
     /// # Safety
     ///
     /// Pointer operations MUST be valid.
-    unsafe fn from_raw(item: *const Self::Item) -> Self;
+    unsafe fn from_ptr(item: *const Self::Item) -> Self;
 
-    fn into_raw(self) -> *const Self::Item;
+    fn into_ptr(self) -> *const Self::Item;
 
     fn as_ptr(&self) -> *const Self::Item;
 }
@@ -57,12 +57,12 @@ unsafe impl<'a, T: ?Sized + Debug> Pointer for &'a T {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> &'a T {
+    unsafe fn from_ptr(raw: *const T) -> &'a T {
         &*raw
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         self
     }
 
@@ -76,12 +76,12 @@ unsafe impl<'a, T: ?Sized + Debug> Pointer for Pin<&'a T> {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> Pin<&'a T> {
+    unsafe fn from_ptr(raw: *const T) -> Pin<&'a T> {
         Pin::new_unchecked(&*raw)
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         unsafe { Pin::into_inner_unchecked(self) as *const T }
     }
 
@@ -94,11 +94,11 @@ unsafe impl<'a, T: ?Sized + Debug> Pointer for Pin<&'a T> {
 unsafe impl<T: ?Sized + Debug> Pointer for NonNull<T> {
     type Item = T;
 
-    unsafe fn from_raw(raw: *const T) -> NonNull<T> {
+    unsafe fn from_ptr(raw: *const T) -> NonNull<T> {
         NonNull::new_unchecked(raw as *mut _)
     }
 
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         self.as_ptr()
     }
 
@@ -112,12 +112,12 @@ unsafe impl<T: ?Sized + Debug> Pointer for Box<T> {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> Box<T> {
+    unsafe fn from_ptr(raw: *const T) -> Box<T> {
         Box::from_raw(raw as *mut T)
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         Box::into_raw(self) as *const T
     }
 
@@ -131,12 +131,12 @@ unsafe impl<T: ?Sized + Debug> Pointer for Pin<Box<T>> {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> Pin<Box<T>> {
+    unsafe fn from_ptr(raw: *const T) -> Pin<Box<T>> {
         Pin::new_unchecked(Box::from_raw(raw as *mut T))
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as *const T
     }
 
@@ -150,12 +150,12 @@ unsafe impl<T: ?Sized + Debug> Pointer for Rc<T> {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> Rc<T> {
+    unsafe fn from_ptr(raw: *const T) -> Rc<T> {
         Rc::from_raw(raw)
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         Rc::into_raw(self)
     }
 
@@ -169,12 +169,12 @@ unsafe impl<T: ?Sized + Debug> Pointer for Pin<Rc<T>> {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> Pin<Rc<T>> {
+    unsafe fn from_ptr(raw: *const T) -> Pin<Rc<T>> {
         Pin::new_unchecked(Rc::from_raw(raw))
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         Rc::into_raw(unsafe { Pin::into_inner_unchecked(self) })
     }
 
@@ -188,12 +188,12 @@ unsafe impl<T: ?Sized + Debug> Pointer for Arc<T> {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> Arc<T> {
+    unsafe fn from_ptr(raw: *const T) -> Arc<T> {
         Arc::from_raw(raw)
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         Arc::into_raw(self)
     }
 
@@ -207,12 +207,12 @@ unsafe impl<T: ?Sized + Debug> Pointer for Pin<Arc<T>> {
     type Item = T;
 
     #[inline]
-    unsafe fn from_raw(raw: *const T) -> Pin<Arc<T>> {
+    unsafe fn from_ptr(raw: *const T) -> Pin<Arc<T>> {
         Pin::new_unchecked(Arc::from_raw(raw))
     }
 
     #[inline]
-    fn into_raw(self) -> *const T {
+    fn into_ptr(self) -> *const T {
         Arc::into_raw(unsafe { Pin::into_inner_unchecked(self) })
     }
 
@@ -265,12 +265,12 @@ mod tests {
                 // Prevent shared pointers from being released by converting them
                 // back into the raw pointers
                 // SAFETY: `pointer` is never dropped. `ManuallyDrop::take` is not stable until 1.42.0.
-                let _ = T::into_raw(unsafe { core::ptr::read(&*self.pointer) });
+                let _ = T::into_ptr(unsafe { core::ptr::read(&*self.pointer) });
             }
         }
 
         let holder = PointerGuard {
-            pointer: ManuallyDrop::new(T::from_raw(ptr)),
+            pointer: ManuallyDrop::new(T::from_ptr(ptr)),
         };
         holder.pointer.deref().clone()
     }
@@ -280,9 +280,9 @@ mod tests {
         unsafe {
             let p = Box::new(1);
             let a: *const i32 = &*p;
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
-            let p2: Box<i32> = <Box<i32> as Pointer>::from_raw(r);
+            let p2: Box<i32> = <Box<i32> as Pointer>::from_ptr(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -293,9 +293,9 @@ mod tests {
         unsafe {
             let p = Rc::new(1);
             let a: *const i32 = &*p;
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
-            let p2: Rc<i32> = <Rc<_> as Pointer>::from_raw(r);
+            let p2: Rc<i32> = <Rc<_> as Pointer>::from_ptr(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -306,9 +306,9 @@ mod tests {
         unsafe {
             let p = Arc::new(1);
             let a: *const i32 = &*p;
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
-            let p2: Arc<i32> = <Arc<_> as Pointer>::from_raw(r);
+            let p2: Arc<i32> = <Arc<_> as Pointer>::from_ptr(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -320,10 +320,10 @@ mod tests {
             let p = Box::new(1) as Box<dyn Debug>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Box<dyn Debug> = <Box<_> as Pointer>::from_raw(r);
+            let p2: Box<dyn Debug> = <Box<_> as Pointer>::from_ptr(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -336,10 +336,10 @@ mod tests {
             let p = Rc::new(1) as Rc<dyn Debug>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Rc<dyn Debug> = <Rc<_> as Pointer>::from_raw(r);
+            let p2: Rc<dyn Debug> = <Rc<_> as Pointer>::from_ptr(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -352,10 +352,10 @@ mod tests {
             let p = Arc::new(1) as Arc<dyn Debug>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Arc<dyn Debug> = <Arc<_> as Pointer>::from_raw(r);
+            let p2: Arc<dyn Debug> = <Arc<_> as Pointer>::from_ptr(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -387,9 +387,9 @@ mod tests {
         unsafe {
             let p = Pin::new(Box::new(1));
             let a: *const i32 = &*p;
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
-            let p2: Pin<Box<i32>> = <Pin<Box<_>> as Pointer>::from_raw(r);
+            let p2: Pin<Box<i32>> = <Pin<Box<_>> as Pointer>::from_ptr(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -400,9 +400,9 @@ mod tests {
         unsafe {
             let p = Pin::new(Rc::new(1));
             let a: *const i32 = &*p;
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
-            let p2: Pin<Rc<i32>> = <Pin<Rc<_>> as Pointer>::from_raw(r);
+            let p2: Pin<Rc<i32>> = <Pin<Rc<_>> as Pointer>::from_ptr(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -413,9 +413,9 @@ mod tests {
         unsafe {
             let p = Pin::new(Arc::new(1));
             let a: *const i32 = &*p;
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
-            let p2: Pin<Arc<i32>> = <Pin<Arc<_>> as Pointer>::from_raw(r);
+            let p2: Pin<Arc<i32>> = <Pin<Arc<_>> as Pointer>::from_ptr(r);
             let a2: *const i32 = &*p2;
             assert_eq!(a, a2);
         }
@@ -427,10 +427,10 @@ mod tests {
             let p = Pin::new(Box::new(1)) as Pin<Box<dyn Debug>>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Pin<Box<dyn Debug>> = <Pin<Box<_>> as Pointer>::from_raw(r);
+            let p2: Pin<Box<dyn Debug>> = <Pin<Box<_>> as Pointer>::from_ptr(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -443,10 +443,10 @@ mod tests {
             let p = Pin::new(Rc::new(1)) as Pin<Rc<dyn Debug>>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Pin<Rc<dyn Debug>> = <Pin<Rc<_>> as Pointer>::from_raw(r);
+            let p2: Pin<Rc<dyn Debug>> = <Pin<Rc<_>> as Pointer>::from_ptr(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -459,10 +459,10 @@ mod tests {
             let p = Pin::new(Arc::new(1)) as Pin<Arc<dyn Debug>>;
             let a: *const dyn Debug = &*p;
             let b: (usize, usize) = mem::transmute(a);
-            let r = p.into_raw();
+            let r = p.into_ptr();
             assert_eq!(a, r);
             assert_eq!(b, mem::transmute(r));
-            let p2: Pin<Arc<dyn Debug>> = <Pin<Arc<_>> as Pointer>::from_raw(r);
+            let p2: Pin<Arc<dyn Debug>> = <Pin<Arc<_>> as Pointer>::from_ptr(r);
             let a2: *const dyn Debug = &*p2;
             assert_eq!(a, a2);
             assert_eq!(b, mem::transmute(a2));
@@ -473,9 +473,9 @@ mod tests {
     fn clone_pin_arc_from_raw() {
         unsafe {
             let p = Pin::new(Arc::new(1));
-            let raw = p.into_raw();
+            let raw = p.into_ptr();
             let p2: Pin<Arc<i32>> = clone_pointer_from_raw(raw);
-            let _p = <Pin<Arc<_>> as Pointer>::from_raw(raw);
+            let _p = <Pin<Arc<_>> as Pointer>::from_ptr(raw);
             assert_eq!(2, Arc::strong_count(&Pin::into_inner(p2)));
         }
     }
@@ -484,9 +484,9 @@ mod tests {
     fn clone_pin_rc_from_raw() {
         unsafe {
             let p = Pin::new(Rc::new(1));
-            let raw = p.into_raw();
+            let raw = p.into_ptr();
             let p2: Pin<Rc<i32>> = clone_pointer_from_raw(raw);
-            let _p = <Pin<Rc<_>> as Pointer>::from_raw(raw);
+            let _p = <Pin<Rc<_>> as Pointer>::from_ptr(raw);
             assert_eq!(2, Rc::strong_count(&Pin::into_inner(p2)));
         }
     }
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 5286f77a..19f8179b 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -111,7 +111,7 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
@@ -136,7 +136,7 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item.as_ptr())
+            A::Pointer::from_ptr(item.as_ptr())
         }
     }
 
@@ -179,7 +179,7 @@ where
         std::mem::forget(self.ptr.take());
 
         let item = self.fifo.adapter.link2item(link);
-        let ptr = A::Pointer::from_raw(item.as_ptr());
+        let ptr = A::Pointer::from_ptr(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index a7b7a9e7..5ec96767 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -202,7 +202,7 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
@@ -241,7 +241,7 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item.as_ptr())
+            A::Pointer::from_ptr(item.as_ptr())
         }
     }
 
@@ -430,7 +430,7 @@ where
         std::mem::forget(self.ptr.take());
 
         let item = self.lfu.adapter.link2item(link);
-        let ptr = A::Pointer::from_raw(item.as_ptr());
+        let ptr = A::Pointer::from_ptr(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index a602bb16..3a8b5af0 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -126,7 +126,7 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
@@ -158,7 +158,7 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item.as_ptr())
+            A::Pointer::from_ptr(item.as_ptr())
         }
     }
 
@@ -304,7 +304,7 @@ where
         std::mem::forget(self.ptr.take());
 
         let item = self.lru.adapter.link2item(link);
-        let ptr = A::Pointer::from_raw(item.as_ptr());
+        let ptr = A::Pointer::from_ptr(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index 5c057c24..5bfd66a5 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -160,7 +160,7 @@ where
 
     fn insert(&mut self, ptr: A::Pointer) {
         unsafe {
-            let item = NonNull::new_unchecked(A::Pointer::into_raw(ptr) as *mut _);
+            let item = NonNull::new_unchecked(A::Pointer::into_ptr(ptr) as *mut _);
             let mut link = self.adapter.item2link(item);
 
             assert!(!link.as_ref().is_linked());
@@ -193,7 +193,7 @@ where
 
             self.len -= 1;
 
-            A::Pointer::from_raw(item.as_ptr())
+            A::Pointer::from_ptr(item.as_ptr())
         }
     }
 
@@ -262,7 +262,7 @@ where
         std::mem::forget(self.ptr.take());
 
         let item = self.sfifo.adapter.link2item(link);
-        let ptr = A::Pointer::from_raw(item.as_ptr());
+        let ptr = A::Pointer::from_ptr(item.as_ptr());
         self.ptr = ManuallyDrop::new(Some(ptr));
     }
 

From 4834cf8dda3bb05b4ea02514271bd8338780f809 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 4 Mar 2024 12:47:35 +0800
Subject: [PATCH 215/261] chore: add comments max width fmt check (#276)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 rustfmt.toml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rustfmt.toml b/rustfmt.toml
index 6c08e327..fdb6e4ee 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1,3 +1,4 @@
 imports_granularity = "Crate"
 group_imports = "StdExternalCrate"
-tab_spaces = 4
\ No newline at end of file
+tab_spaces = 4
+comment_width = 120
\ No newline at end of file

From 47048bc8800c12ebe7d10bd07b6743c1faefb4dc Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 4 Mar 2024 13:49:51 +0800
Subject: [PATCH 216/261] chore: update code and comments width (#277)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/code.rs                      | 12 +--
 foyer-common/src/erwlock.rs                   |  4 +-
 foyer-common/src/queue.rs                     | 10 +-
 foyer-common/src/range.rs                     | 11 +--
 foyer-common/src/rate.rs                      |  4 +-
 foyer-common/src/rated_ticket.rs              |  8 +-
 foyer-experimental-bench/benches/wal-bench.rs |  7 +-
 foyer-experimental-bench/src/analyze.rs       | 36 ++------
 foyer-experimental-bench/src/export.rs        |  8 +-
 foyer-experimental-bench/src/lib.rs           | 15 +--
 foyer-experimental-bench/src/utils.rs         |  4 +-
 foyer-experimental/src/metrics.rs             | 33 ++-----
 foyer-experimental/src/wal.rs                 | 27 +-----
 foyer-intrusive/src/collections/dlist.rs      | 14 +--
 .../src/collections/duplicated_hashmap.rs     | 11 +--
 foyer-intrusive/src/collections/hashmap.rs    | 12 +--
 foyer-intrusive/src/eviction/fifo.rs          | 10 +-
 foyer-intrusive/src/eviction/lfu.rs           | 11 +--
 foyer-intrusive/src/eviction/lru.rs           | 11 +--
 foyer-intrusive/src/eviction/mod.rs           |  5 +-
 foyer-intrusive/src/eviction/sfifo.rs         |  7 +-
 foyer-intrusive/src/lib.rs                    |  3 +-
 foyer-memory/src/lib.rs                       |  3 +-
 foyer-storage-bench/src/analyze.rs            | 36 ++------
 foyer-storage-bench/src/export.rs             |  8 +-
 foyer-storage-bench/src/main.rs               | 91 +++++--------------
 foyer-storage-bench/src/utils.rs              |  4 +-
 foyer-storage/src/buffer.rs                   | 14 +--
 foyer-storage/src/catalog.rs                  |  8 +-
 foyer-storage/src/device/allocator.rs         | 19 +---
 foyer-storage/src/device/error.rs             |  5 +-
 foyer-storage/src/device/fs.rs                | 18 +---
 foyer-storage/src/device/mod.rs               |  4 +-
 foyer-storage/src/flusher.rs                  | 19 +---
 foyer-storage/src/generic.rs                  | 78 ++++------------
 foyer-storage/src/judge.rs                    |  4 +-
 foyer-storage/src/metrics.rs                  | 38 ++------
 foyer-storage/src/reclaimer.rs                |  4 +-
 foyer-storage/src/region.rs                   |  9 +-
 foyer-storage/src/region_manager.rs           |  4 +-
 foyer-storage/src/runtime.rs                  |  5 +-
 foyer-storage/src/storage.rs                  | 60 +++---------
 foyer-storage/src/store.rs                    | 55 ++++-------
 foyer-storage/tests/storage_test.rs           | 25 +----
 rustfmt.toml                                  |  2 +
 45 files changed, 186 insertions(+), 590 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 6695fb21..7c2cb8ac 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -99,17 +99,7 @@ pub trait Cursor: Send + Sync + 'static + std::io::Read + std::fmt::Debug {
 /// If cloning a [`Key`] is expensive, wrap it with [`std::sync::Arc`].
 #[expect(unused_variables)]
 pub trait Key:
-    Sized
-    + Send
-    + Sync
-    + 'static
-    + std::hash::Hash
-    + Eq
-    + PartialEq
-    + Ord
-    + PartialOrd
-    + std::fmt::Debug
-    + Clone
+    Sized + Send + Sync + 'static + std::hash::Hash + Eq + PartialEq + Ord + PartialOrd + std::fmt::Debug + Clone
 {
     type Cursor: Cursor<T = Self> = UnimplementedCursor<Self>;
 
diff --git a/foyer-common/src/erwlock.rs b/foyer-common/src/erwlock.rs
index 2cee08b1..bc820901 100644
--- a/foyer-common/src/erwlock.rs
+++ b/foyer-common/src/erwlock.rs
@@ -14,9 +14,7 @@
 
 use std::sync::Arc;
 
-use parking_lot::{
-    lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard,
-};
+use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 pub trait ErwLockInner {
     type R;
diff --git a/foyer-common/src/queue.rs b/foyer-common/src/queue.rs
index c63e0eda..379365f2 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/queue.rs
@@ -124,14 +124,8 @@ mod tests {
         let queue = AsyncQueue::new();
         let mut read_future1 = pin!(queue.acquire());
         let mut read_future2 = pin!(queue.acquire());
-        assert_eq!(
-            Pending,
-            poll_fn(|cx| Poll::Ready(read_future1.as_mut().poll(cx))).await
-        );
-        assert_eq!(
-            Pending,
-            poll_fn(|cx| Poll::Ready(read_future2.as_mut().poll(cx))).await
-        );
+        assert_eq!(Pending, poll_fn(|cx| Poll::Ready(read_future1.as_mut().poll(cx))).await);
+        assert_eq!(Pending, poll_fn(|cx| Poll::Ready(read_future2.as_mut().poll(cx))).await);
         queue.release(1);
         queue.release(2);
         assert_eq!(1, read_future1.await);
diff --git a/foyer-common/src/range.rs b/foyer-common/src/range.rs
index beef3900..c986c827 100644
--- a/foyer-common/src/range.rs
+++ b/foyer-common/src/range.rs
@@ -48,15 +48,8 @@ mod private {
 
 use private::ZeroOne;
 
-pub trait Idx = PartialOrd<Self>
-    + Add<Output = Self>
-    + Sub<Output = Self>
-    + Clone
-    + Copy
-    + Send
-    + Sync
-    + 'static
-    + ZeroOne;
+pub trait Idx =
+    PartialOrd<Self> + Add<Output = Self> + Sub<Output = Self> + Clone + Copy + Send + Sync + 'static + ZeroOne;
 
 pub trait RangeBoundsExt<T: Idx>: RangeBounds<T> {
     fn start(&self) -> Option<T> {
diff --git a/foyer-common/src/rate.rs b/foyer-common/src/rate.rs
index fdb55ce2..46d6105f 100644
--- a/foyer-common/src/rate.rs
+++ b/foyer-common/src/rate.rs
@@ -105,9 +105,7 @@ mod tests {
             handle.join().unwrap();
         }
 
-        let error = (v.load(Ordering::Relaxed) as isize
-            - RATE as isize * DURATION.as_secs() as isize)
-            .unsigned_abs();
+        let error = (v.load(Ordering::Relaxed) as isize - RATE as isize * DURATION.as_secs() as isize).unsigned_abs();
         let eratio = error as f64 / (RATE as f64 * DURATION.as_secs_f64());
         assert!(eratio < ERATIO, "eratio: {}, target: {}", eratio, ERATIO);
     }
diff --git a/foyer-common/src/rated_ticket.rs b/foyer-common/src/rated_ticket.rs
index 4673f2e0..078ccaf4 100644
--- a/foyer-common/src/rated_ticket.rs
+++ b/foyer-common/src/rated_ticket.rs
@@ -108,9 +108,7 @@ mod tests {
         const CASES: usize = 10;
         const ERATIO: f64 = 0.05;
 
-        let handles = (0..CASES)
-            .map(|_| std::thread::spawn(move || case(f)))
-            .collect_vec();
+        let handles = (0..CASES).map(|_| std::thread::spawn(move || case(f))).collect_vec();
         let mut eratios = vec![];
         for handle in handles {
             let eratio = handle.join().unwrap();
@@ -174,9 +172,7 @@ mod tests {
             handle.join().unwrap();
         }
 
-        let error = (v.load(Ordering::Relaxed) as isize
-            - RATE as isize * DURATION.as_secs() as isize)
-            .unsigned_abs();
+        let error = (v.load(Ordering::Relaxed) as isize - RATE as isize * DURATION.as_secs() as isize).unsigned_abs();
         error as f64 / (RATE as f64 * DURATION.as_secs_f64())
     }
 }
diff --git a/foyer-experimental-bench/benches/wal-bench.rs b/foyer-experimental-bench/benches/wal-bench.rs
index bc9470ba..1eed2650 100644
--- a/foyer-experimental-bench/benches/wal-bench.rs
+++ b/foyer-experimental-bench/benches/wal-bench.rs
@@ -107,12 +107,7 @@ async fn write(log: TombstoneLog<u64>, args: Args, rt: Arc<BackgroundShutdownRun
     let start = Instant::now();
     let mut log = log;
 
-    let mut rng = StdRng::seed_from_u64(
-        SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .unwrap()
-            .as_nanos() as _,
-    );
+    let mut rng = StdRng::seed_from_u64(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() as _);
     loop {
         if start.elapsed() >= Duration::from_secs(args.time as _) {
             return;
diff --git a/foyer-experimental-bench/src/analyze.rs b/foyer-experimental-bench/src/analyze.rs
index bf9190dc..05db6137 100644
--- a/foyer-experimental-bench/src/analyze.rs
+++ b/foyer-experimental-bench/src/analyze.rs
@@ -130,19 +130,13 @@ impl Default for Metrics {
         Self {
             insert_ios: Arc::new(AtomicUsize::new(0)),
             insert_bytes: Arc::new(AtomicUsize::new(0)),
-            insert_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
+            insert_lats: Arc::new(RwLock::new(Histogram::new_with_bounds(1, 10_000_000, 2).unwrap())),
 
             get_ios: Arc::new(AtomicUsize::new(0)),
             get_bytes: Arc::new(AtomicUsize::new(0)),
             get_miss_ios: Arc::new(AtomicUsize::new(0)),
-            get_hit_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
-            get_miss_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
+            get_hit_lats: Arc::new(RwLock::new(Histogram::new_with_bounds(1, 10_000_000, 2).unwrap())),
+            get_miss_lats: Arc::new(RwLock::new(Histogram::new_with_bounds(1, 10_000_000, 2).unwrap())),
         }
     }
 }
@@ -192,22 +186,14 @@ impl std::fmt::Display for Analysis {
         let disk_total_throughput = disk_read_throughput + disk_write_throughput;
 
         // disk statics
-        writeln!(
-            f,
-            "disk total iops: {:.1}",
-            self.disk_write_iops + self.disk_read_iops
-        )?;
+        writeln!(f, "disk total iops: {:.1}", self.disk_write_iops + self.disk_read_iops)?;
         writeln!(
             f,
             "disk total throughput: {}/s",
             disk_total_throughput.to_string_as(true)
         )?;
         writeln!(f, "disk read iops: {:.1}", self.disk_read_iops)?;
-        writeln!(
-            f,
-            "disk read throughput: {}/s",
-            disk_read_throughput.to_string_as(true)
-        )?;
+        writeln!(f, "disk read throughput: {}/s", disk_read_throughput.to_string_as(true))?;
         writeln!(f, "disk write iops: {:.1}", self.disk_write_iops)?;
         writeln!(
             f,
@@ -218,11 +204,7 @@ impl std::fmt::Display for Analysis {
         // insert statics
         let insert_throughput = ByteSize::b(self.insert_throughput as u64);
         writeln!(f, "insert iops: {:.1}/s", self.insert_iops)?;
-        writeln!(
-            f,
-            "insert throughput: {}/s",
-            insert_throughput.to_string_as(true)
-        )?;
+        writeln!(f, "insert throughput: {}/s", insert_throughput.to_string_as(true))?;
         writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
         writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
         writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
@@ -264,15 +246,13 @@ pub fn analyze(
 ) -> Analysis {
     let secs = duration.as_secs_f64();
     let disk_read_iops = (iostat_end.read_ios - iostat_start.read_ios) as f64 / secs;
-    let disk_read_throughput =
-        (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+    let disk_read_throughput = (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
     let disk_write_iops = (iostat_end.write_ios - iostat_start.write_ios) as f64 / secs;
     let disk_write_throughput =
         (iostat_end.write_sectors - iostat_start.write_sectors) as f64 * SECTOR_SIZE as f64 / secs;
 
     let insert_iops = (metrics_dump_end.insert_ios - metrics_dump_start.insert_ios) as f64 / secs;
-    let insert_throughput =
-        (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
+    let insert_throughput = (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
 
     let get_iops = (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64 / secs;
     let get_miss = (metrics_dump_end.get_miss_ios - metrics_dump_start.get_miss_ios) as f64
diff --git a/foyer-experimental-bench/src/export.rs b/foyer-experimental-bench/src/export.rs
index 2d918221..0659d979 100644
--- a/foyer-experimental-bench/src/export.rs
+++ b/foyer-experimental-bench/src/export.rs
@@ -43,11 +43,9 @@ impl MetricsExporter {
                 };
                 let io = hyper_util::rt::TokioIo::new(stream);
                 tokio::spawn(async move {
-                    if let Err(e) = hyper_util::server::conn::auto::Builder::new(
-                        hyper_util::rt::TokioExecutor::new(),
-                    )
-                    .serve_connection(io, service_fn(Self::serve))
-                    .await
+                    if let Err(e) = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new())
+                        .serve_connection(io, service_fn(Self::serve))
+                        .await
                     {
                         tracing::error!("Prometheus service error: {}", e);
                     }
diff --git a/foyer-experimental-bench/src/lib.rs b/foyer-experimental-bench/src/lib.rs
index a4ee5955..c23b1788 100644
--- a/foyer-experimental-bench/src/lib.rs
+++ b/foyer-experimental-bench/src/lib.rs
@@ -42,11 +42,10 @@ pub fn init_logger() {
     use tracing::Level;
     use tracing_subscriber::{filter::Targets, prelude::*};
 
-    let trace_config =
-        Config::default().with_resource(Resource::new(vec![opentelemetry::KeyValue::new(
-            SERVICE_NAME,
-            "foyer-storage-bench",
-        )]));
+    let trace_config = Config::default().with_resource(Resource::new(vec![opentelemetry::KeyValue::new(
+        SERVICE_NAME,
+        "foyer-storage-bench",
+    )]));
     let batch_config = BatchConfig::default()
         .with_max_queue_size(1048576)
         .with_max_export_batch_size(4096)
@@ -133,11 +132,7 @@ pub fn io_monitor(config: IoMonitorConfig) -> IoMonitorGuard {
                     &last_metrics_dump,
                     &new_metrics_dump,
                 );
-                println!(
-                    "Report [{}s/{}s]",
-                    start.elapsed().as_secs(),
-                    config.total_secs
-                );
+                println!("Report [{}s/{}s]", start.elapsed().as_secs(), config.total_secs);
                 println!("{}", analysis);
                 last_io_stat = new_io_stat;
                 last_metrics_dump = new_metrics_dump;
diff --git a/foyer-experimental-bench/src/utils.rs b/foyer-experimental-bench/src/utils.rs
index 2770a57c..84ad7ce9 100644
--- a/foyer-experimental-bench/src/utils.rs
+++ b/foyer-experimental-bench/src/utils.rs
@@ -45,9 +45,7 @@ pub enum FsType {
 pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
     #[cfg(target_os = "linux")]
     {
-        use nix::sys::statfs::{
-            statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC,
-        };
+        use nix::sys::statfs::{statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC};
         let fs_stat = statfs(path.as_ref()).unwrap();
         match fs_stat.filesystem_type() {
             XFS_SUPER_MAGIC => FsType::Xfs,
diff --git a/foyer-experimental/src/metrics.rs b/foyer-experimental/src/metrics.rs
index d8860e11..2d1d9920 100644
--- a/foyer-experimental/src/metrics.rs
+++ b/foyer-experimental/src/metrics.rs
@@ -16,9 +16,8 @@ use std::sync::{LazyLock, OnceLock};
 
 use prometheus::{
     core::{AtomicU64, GenericGauge, GenericGaugeVec},
-    exponential_buckets, opts, register_histogram_vec_with_registry,
-    register_int_counter_vec_with_registry, Histogram, HistogramVec, IntCounter, IntCounterVec,
-    Registry,
+    exponential_buckets, opts, register_histogram_vec_with_registry, register_int_counter_vec_with_registry, Histogram,
+    HistogramVec, IntCounter, IntCounterVec, Registry,
 };
 
 type UintGaugeVec = GenericGaugeVec<AtomicU64>;
@@ -27,9 +26,7 @@ type UintGauge = GenericGauge<AtomicU64>;
 macro_rules! register_gauge_vec {
     ($TYPE:ident, $OPTS:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
         let gauge_vec = $TYPE::new($OPTS, $LABELS_NAMES).unwrap();
-        $REGISTRY
-            .register(Box::new(gauge_vec.clone()))
-            .map(|_| gauge_vec)
+        $REGISTRY.register(Box::new(gauge_vec.clone())).map(|_| gauge_vec)
     }};
 }
 
@@ -160,30 +157,18 @@ pub struct Metrics {
 
 impl Metrics {
     pub fn new(global: &GlobalMetrics, foyer: &str) -> Self {
-        let inner_op_duration_wal_flush = global
-            .inner_op_duration
-            .with_label_values(&[foyer, "wal", "flush"]);
-        let inner_op_bytes_wal_flush = global
-            .inner_op_bytes
-            .with_label_values(&[foyer, "wal", "flush"]);
+        let inner_op_duration_wal_flush = global.inner_op_duration.with_label_values(&[foyer, "wal", "flush"]);
+        let inner_op_bytes_wal_flush = global.inner_op_bytes.with_label_values(&[foyer, "wal", "flush"]);
         let inner_op_bytes_distribution_wal_flush = global
             .inner_op_bytes_distribution
             .with_label_values(&[foyer, "wal", "flush"]);
 
-        let inner_op_duration_wal_write = global
-            .inner_op_duration
-            .with_label_values(&[foyer, "wal", "write"]);
+        let inner_op_duration_wal_write = global.inner_op_duration.with_label_values(&[foyer, "wal", "write"]);
 
-        let inner_op_duration_wal_sync = global
-            .inner_op_duration
-            .with_label_values(&[foyer, "wal", "sync"]);
+        let inner_op_duration_wal_sync = global.inner_op_duration.with_label_values(&[foyer, "wal", "sync"]);
 
-        let inner_op_duration_wal_append = global
-            .inner_op_duration
-            .with_label_values(&[foyer, "wal", "append"]);
-        let inner_op_duration_wal_notify = global
-            .inner_op_duration
-            .with_label_values(&[foyer, "wal", "notify"]);
+        let inner_op_duration_wal_append = global.inner_op_duration.with_label_values(&[foyer, "wal", "append"]);
+        let inner_op_duration_wal_notify = global.inner_op_duration.with_label_values(&[foyer, "wal", "notify"]);
 
         let total_bytes = global.total_bytes.with_label_values(&[foyer, "", ""]);
 
diff --git a/foyer-experimental/src/wal.rs b/foyer-experimental/src/wal.rs
index 49c62839..ba33ed2a 100644
--- a/foyer-experimental/src/wal.rs
+++ b/foyer-experimental/src/wal.rs
@@ -147,11 +147,7 @@ impl<H: HashValue> TombstoneLog<H> {
 
         path.push(format!("tombstone-{:08X}", config.id));
 
-        let file = OpenOptions::new()
-            .write(true)
-            .read(true)
-            .create(true)
-            .open(path)?;
+        let file = OpenOptions::new().write(true).read(true).create(true).open(path)?;
 
         let inner = Arc::new(TombstoneLogInner {
             inflights: Mutex::new(vec![]),
@@ -267,12 +263,7 @@ impl<H: HashValue> TombstoneLogFlusher<H> {
             match self.file.write_all(&buffer) {
                 Ok(()) => {}
                 Err(e) => {
-                    self.task_tx
-                        .send(FlushNotifyTask {
-                            txs,
-                            io_result: Err(e),
-                        })
-                        .unwrap();
+                    self.task_tx.send(FlushNotifyTask { txs, io_result: Err(e) }).unwrap();
                     continue;
                 }
             }
@@ -282,23 +273,13 @@ impl<H: HashValue> TombstoneLogFlusher<H> {
             match self.file.sync_data() {
                 Ok(()) => {}
                 Err(e) => {
-                    self.task_tx
-                        .send(FlushNotifyTask {
-                            txs,
-                            io_result: Err(e),
-                        })
-                        .unwrap();
+                    self.task_tx.send(FlushNotifyTask { txs, io_result: Err(e) }).unwrap();
                     continue;
                 }
             }
             drop(timer_sync);
 
-            self.task_tx
-                .send(FlushNotifyTask {
-                    txs,
-                    io_result: Ok(()),
-                })
-                .unwrap();
+            self.task_tx.send(FlushNotifyTask { txs, io_result: Ok(()) }).unwrap();
 
             drop(timer);
         }
diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index c3c6124b..ac224170 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -542,20 +542,12 @@ mod tests {
             Self
         }
 
-        unsafe fn link2item(
-            &self,
-            link: NonNull<Self::Link>,
-        ) -> NonNull<<Self::Pointer as Pointer>::Item> {
+        unsafe fn link2item(&self, link: NonNull<Self::Link>) -> NonNull<<Self::Pointer as Pointer>::Item> {
             NonNull::new_unchecked(crate::container_of!(link.as_ptr(), DlistItem, link) as *mut _)
         }
 
-        unsafe fn item2link(
-            &self,
-            item: NonNull<<Self::Pointer as Pointer>::Item>,
-        ) -> NonNull<Self::Link> {
-            NonNull::new_unchecked(
-                (item.as_ptr() as *const u8).add(crate::offset_of!(DlistItem, link)) as *mut _,
-            )
+        unsafe fn item2link(&self, item: NonNull<<Self::Pointer as Pointer>::Item>) -> NonNull<Self::Link> {
+            NonNull::new_unchecked((item.as_ptr() as *const u8).add(crate::offset_of!(DlistItem, link)) as *mut _)
         }
     }
 
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index ec803f44..e0a541fa 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -217,10 +217,7 @@ where
     /// # Safety
     ///
     /// `link` MUST be in this [`HashMap`].
-    pub unsafe fn remove_in_place(
-        &mut self,
-        mut link: NonNull<DuplicatedHashMapLink>,
-    ) -> A::Pointer {
+    pub unsafe fn remove_in_place(&mut self, mut link: NonNull<DuplicatedHashMapLink>) -> A::Pointer {
         assert!(link.as_ref().is_linked());
         let item = self.adapter.link2item(link);
         let key = self.adapter.item2key(item).as_ref();
@@ -300,11 +297,7 @@ where
     /// # Safety
     ///
     /// there must be at most one matches in the slot
-    unsafe fn lookup_inner(
-        &self,
-        key: &K,
-        slot: usize,
-    ) -> Option<DlistIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
+    unsafe fn lookup_inner(&self, key: &K, slot: usize) -> Option<DlistIter<'_, DuplicatedHashMapLinkSlotAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 3e1f1969..0c5091b6 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -185,11 +185,7 @@ where
     /// # Safety
     ///
     /// there must be at most one matches in the slot
-    unsafe fn lookup_inner_mut(
-        &mut self,
-        key: &K,
-        slot: usize,
-    ) -> Option<DlistIterMut<'_, HashMapLinkAdapter>> {
+    unsafe fn lookup_inner_mut(&mut self, key: &K, slot: usize) -> Option<DlistIterMut<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter_mut();
         iter.front();
         while iter.is_valid() {
@@ -206,11 +202,7 @@ where
     /// # Safety
     ///
     /// there must be at most one matches in the slot
-    unsafe fn lookup_inner(
-        &self,
-        key: &K,
-        slot: usize,
-    ) -> Option<DlistIter<'_, HashMapLinkAdapter>> {
+    unsafe fn lookup_inner(&self, key: &K, slot: usize) -> Option<DlistIter<'_, HashMapLinkAdapter>> {
         let mut iter = self.slots[slot].iter();
         iter.front();
         while iter.is_valid() {
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index 19f8179b..b3615299 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -129,10 +129,7 @@ where
 
             assert!(link.as_ref().is_linked());
 
-            self.queue
-                .iter_mut_from_raw(link.as_ref().link.raw())
-                .remove()
-                .unwrap();
+            self.queue.iter_mut_from_raw(link.as_ref().link.raw()).remove().unwrap();
 
             self.len -= 1;
 
@@ -324,10 +321,7 @@ mod tests {
         let v = fifo.iter().map(|item| item.key).collect_vec();
         assert_eq!(v, (0..10).collect_vec());
 
-        let v = (0..5)
-            .map(|_| fifo.pop().unwrap())
-            .map(|item| item.key)
-            .collect_vec();
+        let v = (0..5).map(|_| fifo.pop().unwrap()).map(|item| item.key).collect_vec();
         assert_eq!(v, (0..5).collect_vec());
 
         let v = fifo.iter().map(|item| item.key).collect_vec();
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 5ec96767..508fdef2 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -213,9 +213,8 @@ where
             self.update_frequencies(link);
 
             // If tiny cache is full, unconditionally promote tail to main cache.
-            let expected_tiny_len = (self.config.tiny_lru_capacity_ratio
-                * (self.lru_tiny.len() + self.lru_main.len()) as f64)
-                as usize;
+            let expected_tiny_len =
+                (self.config.tiny_lru_capacity_ratio * (self.lru_tiny.len() + self.lru_main.len()) as f64) as usize;
             if self.lru_tiny.len() > expected_tiny_len {
                 let raw = self.lru_tiny.back().unwrap().raw();
                 self.switch_to_lru_front(raw);
@@ -614,11 +613,7 @@ mod tests {
         assert_eq!(items[100].link.lru_type(), LruType::Tiny);
 
         assert_eq!(
-            [100]
-                .into_iter()
-                .chain(1..100)
-                .chain([0].into_iter())
-                .collect_vec(),
+            [100].into_iter().chain(1..100).chain([0].into_iter()).collect_vec(),
             lfu.iter().map(|item| item.key).collect_vec()
         );
 
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 3a8b5af0..25ccdaf8 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -216,20 +216,15 @@ where
 
             assert!(self.insertion_point.is_some());
 
-            let expected_tail_len =
-                (self.lru.len() as f64 * (1.0 - self.config.lru_insertion_point_fraction)) as usize;
+            let expected_tail_len = (self.lru.len() as f64 * (1.0 - self.config.lru_insertion_point_fraction)) as usize;
 
             let mut curr = self.insertion_point.unwrap();
-            while self.tail_len < expected_tail_len
-                && Some(curr) != self.lru.front().map(LruLink::raw)
-            {
+            while self.tail_len < expected_tail_len && Some(curr) != self.lru.front().map(LruLink::raw) {
                 curr = self.lru_prev(curr).unwrap();
                 curr.as_mut().is_in_tail = true;
                 self.tail_len += 1;
             }
-            while self.tail_len > expected_tail_len
-                && Some(curr) != self.lru.back().map(LruLink::raw)
-            {
+            while self.tail_len > expected_tail_len && Some(curr) != self.lru.back().map(LruLink::raw) {
                 curr.as_mut().is_in_tail = false;
                 self.tail_len -= 1;
                 curr = self.lru_next(curr).unwrap();
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index 4338532c..6055040e 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -26,10 +26,7 @@ pub trait EvictionPolicy: Send + Sync + Debug + 'static {
 
     fn insert(&mut self, ptr: <Self::Adapter as Adapter>::Pointer);
 
-    fn remove(
-        &mut self,
-        ptr: &<Self::Adapter as Adapter>::Pointer,
-    ) -> <Self::Adapter as Adapter>::Pointer;
+    fn remove(&mut self, ptr: &<Self::Adapter as Adapter>::Pointer) -> <Self::Adapter as Adapter>::Pointer;
 
     fn access(&mut self, ptr: &<Self::Adapter as Adapter>::Pointer);
 
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index 5bfd66a5..c0db363f 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -141,9 +141,7 @@ where
     A::Pointer: Clone,
 {
     pub fn new(config: SegmentedFifoConfig) -> Self {
-        let segments = (0..config.segment_ratios.len())
-            .map(|_| Dlist::new())
-            .collect_vec();
+        let segments = (0..config.segment_ratios.len()).map(|_| Dlist::new()).collect_vec();
         let total_ratio = config.segment_ratios.iter().sum();
 
         Self {
@@ -215,8 +213,7 @@ where
             //      and a segment ratio of [1, 1, 1]
             for high in (1..self.segments.len()).rev() {
                 let low = high - 1;
-                let limit = (total as f64 * self.config.segment_ratios[high] as f64
-                    / self.total_ratio as f64) as usize;
+                let limit = (total as f64 * self.config.segment_ratios[high] as f64 / self.total_ratio as f64) as usize;
                 while self.segments[high].len() > limit {
                     let mut link = self.segments[high].pop_front().unwrap();
                     link.as_mut().priority = low;
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 19104d42..2393da1b 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -42,8 +42,7 @@ pub use memoffset::offset_of;
 #[macro_export]
 macro_rules! container_of {
     ($ptr:expr, $container:path, $field:ident) => {
-        ($ptr as *const _ as *const u8).sub($crate::offset_of!($container, $field))
-            as *const $container
+        ($ptr as *const _ as *const u8).sub($crate::offset_of!($container, $field)) as *const $container
     };
 }
 
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index e695f93e..0e0dce70 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -228,8 +228,7 @@ mod tests {
     }
 
     type FifoCacheConfig = CacheConfig<SegmentedFifo<CacheItemEpAdapter<K, V, SegmentedFifoLink>>>;
-    type FifoCache =
-        Cache<K, V, SegmentedFifo<CacheItemEpAdapter<K, V, SegmentedFifoLink>>, SegmentedFifoLink>;
+    type FifoCache = Cache<K, V, SegmentedFifo<CacheItemEpAdapter<K, V, SegmentedFifoLink>>, SegmentedFifoLink>;
 
     #[test]
     fn test_fifo_cache_simple() {
diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index bf9190dc..05db6137 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -130,19 +130,13 @@ impl Default for Metrics {
         Self {
             insert_ios: Arc::new(AtomicUsize::new(0)),
             insert_bytes: Arc::new(AtomicUsize::new(0)),
-            insert_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
+            insert_lats: Arc::new(RwLock::new(Histogram::new_with_bounds(1, 10_000_000, 2).unwrap())),
 
             get_ios: Arc::new(AtomicUsize::new(0)),
             get_bytes: Arc::new(AtomicUsize::new(0)),
             get_miss_ios: Arc::new(AtomicUsize::new(0)),
-            get_hit_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
-            get_miss_lats: Arc::new(RwLock::new(
-                Histogram::new_with_bounds(1, 10_000_000, 2).unwrap(),
-            )),
+            get_hit_lats: Arc::new(RwLock::new(Histogram::new_with_bounds(1, 10_000_000, 2).unwrap())),
+            get_miss_lats: Arc::new(RwLock::new(Histogram::new_with_bounds(1, 10_000_000, 2).unwrap())),
         }
     }
 }
@@ -192,22 +186,14 @@ impl std::fmt::Display for Analysis {
         let disk_total_throughput = disk_read_throughput + disk_write_throughput;
 
         // disk statics
-        writeln!(
-            f,
-            "disk total iops: {:.1}",
-            self.disk_write_iops + self.disk_read_iops
-        )?;
+        writeln!(f, "disk total iops: {:.1}", self.disk_write_iops + self.disk_read_iops)?;
         writeln!(
             f,
             "disk total throughput: {}/s",
             disk_total_throughput.to_string_as(true)
         )?;
         writeln!(f, "disk read iops: {:.1}", self.disk_read_iops)?;
-        writeln!(
-            f,
-            "disk read throughput: {}/s",
-            disk_read_throughput.to_string_as(true)
-        )?;
+        writeln!(f, "disk read throughput: {}/s", disk_read_throughput.to_string_as(true))?;
         writeln!(f, "disk write iops: {:.1}", self.disk_write_iops)?;
         writeln!(
             f,
@@ -218,11 +204,7 @@ impl std::fmt::Display for Analysis {
         // insert statics
         let insert_throughput = ByteSize::b(self.insert_throughput as u64);
         writeln!(f, "insert iops: {:.1}/s", self.insert_iops)?;
-        writeln!(
-            f,
-            "insert throughput: {}/s",
-            insert_throughput.to_string_as(true)
-        )?;
+        writeln!(f, "insert throughput: {}/s", insert_throughput.to_string_as(true))?;
         writeln!(f, "insert lat p50: {}us", self.insert_lat_p50)?;
         writeln!(f, "insert lat p90: {}us", self.insert_lat_p90)?;
         writeln!(f, "insert lat p99: {}us", self.insert_lat_p99)?;
@@ -264,15 +246,13 @@ pub fn analyze(
 ) -> Analysis {
     let secs = duration.as_secs_f64();
     let disk_read_iops = (iostat_end.read_ios - iostat_start.read_ios) as f64 / secs;
-    let disk_read_throughput =
-        (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
+    let disk_read_throughput = (iostat_end.read_sectors - iostat_start.read_sectors) as f64 * SECTOR_SIZE as f64 / secs;
     let disk_write_iops = (iostat_end.write_ios - iostat_start.write_ios) as f64 / secs;
     let disk_write_throughput =
         (iostat_end.write_sectors - iostat_start.write_sectors) as f64 * SECTOR_SIZE as f64 / secs;
 
     let insert_iops = (metrics_dump_end.insert_ios - metrics_dump_start.insert_ios) as f64 / secs;
-    let insert_throughput =
-        (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
+    let insert_throughput = (metrics_dump_end.insert_bytes - metrics_dump_start.insert_bytes) as f64 / secs;
 
     let get_iops = (metrics_dump_end.get_ios - metrics_dump_start.get_ios) as f64 / secs;
     let get_miss = (metrics_dump_end.get_miss_ios - metrics_dump_start.get_miss_ios) as f64
diff --git a/foyer-storage-bench/src/export.rs b/foyer-storage-bench/src/export.rs
index 2d918221..0659d979 100644
--- a/foyer-storage-bench/src/export.rs
+++ b/foyer-storage-bench/src/export.rs
@@ -43,11 +43,9 @@ impl MetricsExporter {
                 };
                 let io = hyper_util::rt::TokioIo::new(stream);
                 tokio::spawn(async move {
-                    if let Err(e) = hyper_util::server::conn::auto::Builder::new(
-                        hyper_util::rt::TokioExecutor::new(),
-                    )
-                    .serve_connection(io, service_fn(Self::serve))
-                    .await
+                    if let Err(e) = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new())
+                        .serve_connection(io, service_fn(Self::serve))
+                        .await
                     {
                         tracing::error!("Prometheus service error: {}", e);
                     }
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index 1a927ad5..fc019f9f 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -190,12 +190,8 @@ where
 {
     fn clone(&self) -> Self {
         match self {
-            Self::StoreConfig { config } => Self::StoreConfig {
-                config: config.clone(),
-            },
-            Self::RuntimeStoreConfig { config } => Self::RuntimeStoreConfig {
-                config: config.clone(),
-            },
+            Self::StoreConfig { config } => Self::StoreConfig { config: config.clone() },
+            Self::RuntimeStoreConfig { config } => Self::RuntimeStoreConfig { config: config.clone() },
         }
     }
 }
@@ -305,12 +301,8 @@ where
 {
     fn clone(&self) -> Self {
         match self {
-            Self::Store { store } => Self::Store {
-                store: store.clone(),
-            },
-            Self::RuntimeStore { store } => Self::RuntimeStore {
-                store: store.clone(),
-            },
+            Self::Store { store } => Self::Store { store: store.clone() },
+            Self::RuntimeStore { store } => Self::RuntimeStore { store: store.clone() },
         }
     }
 }
@@ -327,9 +319,7 @@ where
 
     async fn open(config: Self::Config) -> Result<Self> {
         match config {
-            BenchStoreConfig::StoreConfig { config } => {
-                Store::open(config).await.map(|store| Self::Store { store })
-            }
+            BenchStoreConfig::StoreConfig { config } => Store::open(config).await.map(|store| Self::Store { store }),
             BenchStoreConfig::RuntimeStoreConfig { config } => RuntimeStore::open(config)
                 .await
                 .map(|store| Self::RuntimeStore { store }),
@@ -389,14 +379,8 @@ where
 #[derive(Debug)]
 enum TimeSeriesDistribution {
     None,
-    Uniform {
-        interval: Duration,
-    },
-    Zipf {
-        n: usize,
-        s: f64,
-        interval: Duration,
-    },
+    Uniform { interval: Duration },
+    Zipf { n: usize, s: f64, interval: Duration },
 }
 
 impl TimeSeriesDistribution {
@@ -405,15 +389,15 @@ impl TimeSeriesDistribution {
             "none" => TimeSeriesDistribution::None,
             "uniform" => {
                 // interval = 1 / freq = 1 / (rate / size) = size / rate
-                let interval = ((args.entry_size_min + args.entry_size_max) >> 1) as f64
-                    / (args.w_rate * 1024.0 * 1024.0);
+                let interval =
+                    ((args.entry_size_min + args.entry_size_max) >> 1) as f64 / (args.w_rate * 1024.0 * 1024.0);
                 let interval = Duration::from_secs_f64(interval);
                 TimeSeriesDistribution::Uniform { interval }
             }
             "zipf" => {
                 // interval = 1 / freq = 1 / (rate / size) = size / rate
-                let interval = ((args.entry_size_min + args.entry_size_max) >> 1) as f64
-                    / (args.w_rate * 1024.0 * 1024.0);
+                let interval =
+                    ((args.entry_size_min + args.entry_size_max) >> 1) as f64 / (args.w_rate * 1024.0 * 1024.0);
                 let interval = Duration::from_secs_f64(interval);
                 display_zipf_sample(args.distribution_zipf_n, args.distribution_zipf_s);
                 TimeSeriesDistribution::Zipf {
@@ -455,11 +439,10 @@ fn init_logger() {
     use tracing::Level;
     use tracing_subscriber::{filter::Targets, prelude::*};
 
-    let trace_config =
-        Config::default().with_resource(Resource::new(vec![opentelemetry::KeyValue::new(
-            SERVICE_NAME,
-            "foyer-storage-bench",
-        )]));
+    let trace_config = Config::default().with_resource(Resource::new(vec![opentelemetry::KeyValue::new(
+        SERVICE_NAME,
+        "foyer-storage-bench",
+    )]));
     let batch_config = BatchConfig::default()
         .with_max_queue_size(1048576)
         .with_max_export_batch_size(4096)
@@ -534,10 +517,7 @@ async fn main() {
 
     println!("{:#?}", args);
 
-    assert!(
-        args.lookup_range > 0,
-        "\"--lookup-range\" value must be greater than 0"
-    );
+    assert!(args.lookup_range > 0, "\"--lookup-range\" value must be greater than 0");
 
     create_dir_all(&args.dir).unwrap();
 
@@ -620,9 +600,7 @@ async fn main() {
             },
         }
     } else {
-        BenchStoreConfig::StoreConfig {
-            config: config.into(),
-        }
+        BenchStoreConfig::StoreConfig { config: config.into() }
     };
 
     println!("{config:#?}");
@@ -644,12 +622,7 @@ async fn main() {
         )
     });
 
-    let handle_bench = tokio::spawn(bench(
-        args.clone(),
-        store.clone(),
-        metrics.clone(),
-        stop_tx.clone(),
-    ));
+    let handle_bench = tokio::spawn(bench(args.clone(), store.clone(), metrics.clone(), stop_tx.clone()));
 
     let handle_signal = tokio::spawn(async move {
         tokio::signal::ctrl_c().await.unwrap();
@@ -694,9 +667,7 @@ async fn bench(
         Some(args.r_rate * 1024.0 * 1024.0)
     };
 
-    let counts = (0..args.writers)
-        .map(|_| AtomicU64::default())
-        .collect_vec();
+    let counts = (0..args.writers).map(|_| AtomicU64::default()).collect_vec();
 
     let distribution = TimeSeriesDistribution::new(&args);
 
@@ -712,14 +683,7 @@ async fn bench(
     });
 
     let w_handles = (0..args.writers)
-        .map(|id| {
-            tokio::spawn(write(
-                id as u64,
-                store.clone(),
-                context.clone(),
-                stop_tx.subscribe(),
-            ))
-        })
+        .map(|id| tokio::spawn(write(id as u64, store.clone(), context.clone(), stop_tx.subscribe())))
         .collect_vec();
     let r_handles = (0..args.readers)
         .map(|_| tokio::spawn(read(store.clone(), context.clone(), stop_tx.subscribe())))
@@ -810,9 +774,7 @@ async fn write(
 
             if inserted {
                 ctx.metrics.insert_ios.fetch_add(1, Ordering::Relaxed);
-                ctx.metrics
-                    .insert_bytes
-                    .fetch_add(entry_size, Ordering::Relaxed);
+                ctx.metrics.insert_bytes.fetch_add(entry_size, Ordering::Relaxed);
             }
         };
 
@@ -831,8 +793,7 @@ async fn write(
                 store.insert_async_with_callback(idx, data, callback);
                 let intervals = zipf_intervals.as_ref().unwrap();
 
-                let group = match intervals.binary_search_by_key(&(c as usize % K), |(sum, _)| *sum)
-                {
+                let group = match intervals.binary_search_by_key(&(c as usize % K), |(sum, _)| *sum) {
                     Ok(i) => i,
                     Err(i) => i.min(G - 1),
                 };
@@ -872,8 +833,7 @@ async fn read(
             tokio::time::sleep(Duration::from_millis(1)).await;
             continue;
         }
-        let c =
-            rng.gen_range(std::cmp::max(c_max, context.lookup_range) - context.lookup_range..c_max);
+        let c = rng.gen_range(std::cmp::max(c_max, context.lookup_range) - context.lookup_range..c_max);
         let idx = w + c * step;
 
         let time = Instant::now();
@@ -886,10 +846,7 @@ async fn read(
             if let Err(e) = context.metrics.get_hit_lats.write().record(lat) {
                 tracing::error!("metrics error: {:?}, value: {}", e, lat);
             }
-            context
-                .metrics
-                .get_bytes
-                .fetch_add(entry_size, Ordering::Relaxed);
+            context.metrics.get_bytes.fetch_add(entry_size, Ordering::Relaxed);
 
             if let Some(limiter) = &mut limiter
                 && let Some(wait) = limiter.consume(entry_size as f64)
diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
index cc486d32..4e5a6a37 100644
--- a/foyer-storage-bench/src/utils.rs
+++ b/foyer-storage-bench/src/utils.rs
@@ -45,9 +45,7 @@ pub enum FsType {
 pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
     #[cfg(target_os = "linux")]
     {
-        use nix::sys::statfs::{
-            statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC,
-        };
+        use nix::sys::statfs::{statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC};
         let fs_stat = statfs(path.as_ref()).unwrap();
         match fs_stat.filesystem_type() {
             XFS_SUPER_MAGIC => FsType::Xfs,
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index cbf1c7d3..efdf04a6 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -87,8 +87,7 @@ where
     D: Device,
 {
     pub fn new(device: D) -> Self {
-        let default_buffer_capacity =
-            align_up(device.align(), device.io_size() + device.io_size() / 2);
+        let default_buffer_capacity = align_up(device.align(), device.io_size() + device.io_size() / 2);
         let buffer = device.io_buffer(0, default_buffer_capacity);
         Self {
             buffer,
@@ -117,10 +116,7 @@ where
     /// Flush io buffer if necessary, and reset io buffer to a new region.
     ///
     /// Returns fully flushed entries.
-    pub async fn rotate(
-        &mut self,
-        region: RegionId,
-    ) -> BufferResult<Vec<PositionedEntry<K, V>>, Entry<K, V>> {
+    pub async fn rotate(&mut self, region: RegionId) -> BufferResult<Vec<PositionedEntry<K, V>>, Entry<K, V>> {
         let entries = self.flush().await?;
         debug_assert!(self.buffer.is_empty());
         self.region = Some(region);
@@ -233,8 +229,7 @@ where
                 std::io::copy(&mut vcursor, &mut self.buffer).map_err(DeviceError::from)?;
             }
             Compression::Zstd => {
-                zstd::stream::copy_encode(&mut vcursor, &mut self.buffer, 0)
-                    .map_err(DeviceError::from)?;
+                zstd::stream::copy_encode(&mut vcursor, &mut self.buffer, 0).map_err(DeviceError::from)?;
             }
             Compression::Lz4 => {
                 let mut encoder = lz4::EncoderBuilder::new()
@@ -257,8 +252,7 @@ where
 
         // calculate checksum
         cursor -= compressed_value_len + encoded_key_len;
-        let checksum =
-            checksum(&self.buffer[cursor..cursor + compressed_value_len + encoded_key_len]);
+        let checksum = checksum(&self.buffer[cursor..cursor + compressed_value_len + encoded_key_len]);
 
         // write entry header
         cursor -= EntryHeader::serialized_len();
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 0cf6bf7d..0c0d9ca6 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -103,12 +103,8 @@ where
     V: Value,
 {
     pub fn new(regions: usize, bits: usize, metrics: Arc<Metrics>) -> Self {
-        let infos = (0..1 << bits)
-            .map(|_| RwLock::new(BTreeMap::new()))
-            .collect_vec();
-        let regions = (0..regions)
-            .map(|_| Mutex::new(BTreeMap::new()))
-            .collect_vec();
+        let infos = (0..1 << bits).map(|_| RwLock::new(BTreeMap::new())).collect_vec();
+        let regions = (0..regions).map(|_| Mutex::new(BTreeMap::new())).collect_vec();
         Self {
             bits,
             items: infos,
diff --git a/foyer-storage/src/device/allocator.rs b/foyer-storage/src/device/allocator.rs
index 94902deb..8c43ab7a 100644
--- a/foyer-storage/src/device/allocator.rs
+++ b/foyer-storage/src/device/allocator.rs
@@ -29,24 +29,15 @@ impl AlignedAllocator {
 }
 
 unsafe impl Allocator for AlignedAllocator {
-    fn allocate(
-        &self,
-        layout: std::alloc::Layout,
-    ) -> Result<std::ptr::NonNull<[u8]>, std::alloc::AllocError> {
-        let layout = std::alloc::Layout::from_size_align(
-            layout.size(),
-            bits::align_up(self.align, layout.align()),
-        )
-        .unwrap();
+    fn allocate(&self, layout: std::alloc::Layout) -> Result<std::ptr::NonNull<[u8]>, std::alloc::AllocError> {
+        let layout =
+            std::alloc::Layout::from_size_align(layout.size(), bits::align_up(self.align, layout.align())).unwrap();
         Global.allocate(layout)
     }
 
     unsafe fn deallocate(&self, ptr: std::ptr::NonNull<u8>, layout: std::alloc::Layout) {
-        let layout = std::alloc::Layout::from_size_align(
-            layout.size(),
-            bits::align_up(self.align, layout.align()),
-        )
-        .unwrap();
+        let layout =
+            std::alloc::Layout::from_size_align(layout.size(), bits::align_up(self.align, layout.align())).unwrap();
         Global.deallocate(ptr, layout)
     }
 }
diff --git a/foyer-storage/src/device/error.rs b/foyer-storage/src/device/error.rs
index 01e9e135..e6708507 100644
--- a/foyer-storage/src/device/error.rs
+++ b/foyer-storage/src/device/error.rs
@@ -60,9 +60,6 @@ mod tests {
 
     #[test]
     fn test_error_size() {
-        assert_eq!(
-            std::mem::size_of::<DeviceError>(),
-            std::mem::size_of::<usize>()
-        );
+        assert_eq!(std::mem::size_of::<DeviceError>(), std::mem::size_of::<usize>());
     }
 }
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index 22cc7752..e7ddec56 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -82,13 +82,7 @@ impl Device for FsDevice {
         Self::open(config).await
     }
 
-    async fn write<B>(
-        &self,
-        buf: B,
-        range: impl IoRange,
-        region: RegionId,
-        offset: usize,
-    ) -> (DeviceResult<usize>, B)
+    async fn write<B>(&self, buf: B, range: impl IoRange, region: RegionId, offset: usize) -> (DeviceResult<usize>, B)
     where
         B: IoBuf,
     {
@@ -106,8 +100,7 @@ impl Device for FsDevice {
 
         asyncify(move || {
             let fd = unsafe { BorrowedFd::borrow_raw(fd) };
-            let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[range], offset as i64)
-                .map_err(DeviceError::from);
+            let res = nix::sys::uio::pwrite(fd, &buf.as_ref()[range], offset as i64).map_err(DeviceError::from);
             (res, buf)
         })
         .await
@@ -137,8 +130,7 @@ impl Device for FsDevice {
 
         asyncify(move || {
             let fd = unsafe { BorrowedFd::borrow_raw(fd) };
-            let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[range], offset as i64)
-                .map_err(DeviceError::from);
+            let res = nix::sys::uio::pread(fd, &mut buf.as_mut()[range], offset as i64).map_err(DeviceError::from);
             (res, buf)
         })
         .await
@@ -234,9 +226,7 @@ impl FsDevice {
             io_buffer_allocator,
         };
 
-        Ok(Self {
-            inner: Arc::new(inner),
-        })
+        Ok(Self { inner: Arc::new(inner) })
     }
 
     fn fd(&self, region: RegionId) -> RawFd {
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index c363ee72..5cc71905 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -98,9 +98,7 @@ pub trait DeviceExt: Device {
 
             while range.start + offset < range.end {
                 let len = std::cmp::min(self.io_size(), size - offset);
-                let (res, b) = self
-                    .read(buf, offset..offset + len, region, range.start + offset)
-                    .await;
+                let (res, b) = self.read(buf, offset..offset + len, region, range.start + offset).await;
                 let bytes = res?;
                 offset += bytes;
                 buf = b;
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 8bb2d8fb..102c30f0 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -152,10 +152,7 @@ where
         // current region is full, rotate flush buffer region and retry
 
         // 1. get a clean region
-        let acquire_clean_region_timer = self
-            .metrics
-            .inner_op_duration_acquire_clean_region
-            .start_timer();
+        let acquire_clean_region_timer = self.metrics.inner_op_duration_acquire_clean_region.start_timer();
         let new_region = self
             .region_manager
             .clean_regions()
@@ -171,12 +168,9 @@ where
             self.region_manager.eviction_push(old_region);
         }
 
-        self.metrics.total_bytes.add(
-            self.region_manager
-                .region(&new_region)
-                .device()
-                .region_size() as u64,
-        );
+        self.metrics
+            .total_bytes
+            .add(self.region_manager.region(&new_region).device().region_size() as u64);
 
         // 3. retry write
         let entries = match self.buffer.write(entry).await {
@@ -209,10 +203,7 @@ where
         {
             bytes += len;
             let index = Index::Region {
-                view: self
-                    .region_manager
-                    .region(&region)
-                    .view(offset as u32, len as u32),
+                view: self.region_manager.region(&region).view(offset as u32, len as u32),
             };
             let item = Item::new(sequence, index);
             self.catalog.insert(key, item);
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 41c2cec1..cd27bbea 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -232,23 +232,15 @@ where
             device.clone(),
         ));
 
-        let catalog = Arc::new(Catalog::new(
-            device.regions(),
-            config.catalog_bits,
-            metrics.clone(),
-        ));
+        let catalog = Arc::new(Catalog::new(device.regions(), config.catalog_bits, metrics.clone()));
 
         let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
-        let flusher_stop_rxs = (0..config.flushers)
-            .map(|_| flushers_stop_tx.subscribe())
-            .collect_vec();
+        let flusher_stop_rxs = (0..config.flushers).map(|_| flushers_stop_tx.subscribe()).collect_vec();
         #[expect(clippy::type_complexity)]
         let (flusher_entry_txs, flusher_entry_rxs): (
             Vec<mpsc::UnboundedSender<Entry<K, V>>>,
             Vec<mpsc::UnboundedReceiver<Entry<K, V>>>,
-        ) = (0..config.flushers)
-            .map(|_| mpsc::unbounded_channel())
-            .unzip();
+        ) = (0..config.flushers).map(|_| mpsc::unbounded_channel()).unzip();
 
         let (reclaimers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
         let reclaimer_stop_rxs = (0..config.reclaimers)
@@ -271,9 +263,7 @@ where
             compression: config.compression,
             _marker: PhantomData,
         };
-        let store = Self {
-            inner: Arc::new(inner),
-        };
+        let store = Self { inner: Arc::new(inner) };
 
         let admission_context = AdmissionContext {
             catalog: catalog.clone(),
@@ -418,10 +408,7 @@ where
 
                 let res = match read_entry::<K, V>(buf.as_ref()) {
                     Ok((_key, value)) => {
-                        self.inner
-                            .metrics
-                            .op_bytes_lookup
-                            .inc_by(value.serialized_len() as u64);
+                        self.inner.metrics.op_bytes_lookup.inc_by(value.serialized_len() as u64);
                         Ok(Some(value))
                     }
                     Err(e) => {
@@ -545,11 +532,7 @@ where
     }
 
     #[tracing::instrument(skip(self, value))]
-    async fn apply_writer(
-        &self,
-        mut writer: GenericStoreWriter<K, V, D, EP, EL>,
-        value: V,
-    ) -> Result<bool> {
+    async fn apply_writer(&self, mut writer: GenericStoreWriter<K, V, D, EP, EL>, value: V) -> Result<bool> {
         debug_assert!(!writer.is_inserted);
 
         if !writer.judge() {
@@ -797,9 +780,7 @@ impl EntryHeader {
         let v = buf.get_u32();
         let magic = v & ENTRY_MAGIC_MASK;
         if magic != ENTRY_MAGIC {
-            return Err(
-                anyhow!("magic mismatch, expected: {}, got: {}", ENTRY_MAGIC, magic).into(),
-            );
+            return Err(anyhow!("magic mismatch, expected: {}, got: {}", ENTRY_MAGIC, magic).into());
         }
         let compression = Compression::try_from(v as u8)?;
 
@@ -833,14 +814,12 @@ where
     let value = match header.compression {
         Compression::None => V::read(compressed)?,
         Compression::Zstd => {
-            let mut decompressed =
-                Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
+            let mut decompressed = Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
             zstd::stream::copy_decode(compressed, &mut decompressed).map_err(CodingError::from)?;
             V::read(&decompressed[..])?
         }
         Compression::Lz4 => {
-            let mut decompressed =
-                Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
+            let mut decompressed = Vec::with_capacity((header.value_len + header.value_len / 2) as usize);
             let mut decoder = lz4::Decoder::new(compressed).map_err(CodingError::from)?;
             std::io::copy(&mut decoder, &mut decompressed).map_err(CodingError::from)?;
             let (_r, res) = decoder.finish();
@@ -855,12 +834,7 @@ where
 
     let checksum = checksum(&buf[EntryHeader::serialized_len()..offset]);
     if checksum != header.checksum {
-        return Err(anyhow!(
-            "magic mismatch, expected: {}, got: {}",
-            header.checksum,
-            checksum
-        )
-        .into());
+        return Err(anyhow!("magic mismatch, expected: {}, got: {}", header.checksum, checksum).into());
     }
 
     Ok((key, value))
@@ -918,11 +892,7 @@ where
             return Ok(None);
         }
 
-        let Some(slice) = self
-            .region
-            .load_range(self.cursor..self.cursor + align)
-            .await?
-        else {
+        let Some(slice) = self.region.load_range(self.cursor..self.cursor + align).await? else {
             return Ok(None);
         };
 
@@ -936,9 +906,7 @@ where
         );
 
         let abs_start = self.cursor + EntryHeader::serialized_len() + header.value_len as usize;
-        let abs_end = self.cursor
-            + EntryHeader::serialized_len()
-            + (header.key_len + header.value_len) as usize;
+        let abs_end = self.cursor + EntryHeader::serialized_len() + (header.key_len + header.value_len) as usize;
 
         if abs_start >= abs_end || abs_end > region_size {
             // Double check wrong entry.
@@ -1108,11 +1076,9 @@ mod tests {
         test_utils::JudgeRecorder,
     };
 
-    type TestStore =
-        GenericStore<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
+    type TestStore = GenericStore<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
-    type TestStoreConfig =
-        GenericStoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
+    type TestStoreConfig = GenericStoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
     #[tokio::test]
     #[expect(clippy::identity_op)]
@@ -1123,10 +1089,8 @@ mod tests {
         let tempdir = tempfile::tempdir().unwrap();
 
         let recorder = Arc::new(JudgeRecorder::default());
-        let admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> =
-            vec![recorder.clone()];
-        let reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> =
-            vec![recorder.clone()];
+        let admissions: Vec<Arc<dyn AdmissionPolicy<Key = u64, Value = Vec<u8>>>> = vec![recorder.clone()];
+        let reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = u64, Value = Vec<u8>>>> = vec![recorder.clone()];
 
         let config = TestStoreConfig {
             name: "".to_string(),
@@ -1166,10 +1130,7 @@ mod tests {
 
         for i in 0..21 {
             if remains.contains(&i) {
-                assert_eq!(
-                    store.lookup(&i).await.unwrap().unwrap(),
-                    vec![i as u8; 1 * MB],
-                );
+                assert_eq!(store.lookup(&i).await.unwrap().unwrap(), vec![i as u8; 1 * MB],);
             } else {
                 assert!(store.lookup(&i).await.unwrap().is_none());
             }
@@ -1200,10 +1161,7 @@ mod tests {
 
         for i in 0..21 {
             if remains.contains(&i) {
-                assert_eq!(
-                    store.lookup(&i).await.unwrap().unwrap(),
-                    vec![i as u8; 1 * MB],
-                );
+                assert_eq!(store.lookup(&i).await.unwrap().unwrap(), vec![i as u8; 1 * MB],);
             } else {
                 assert!(store.lookup(&i).await.unwrap().is_none());
             }
diff --git a/foyer-storage/src/judge.rs b/foyer-storage/src/judge.rs
index 3b1e121d..695085c4 100644
--- a/foyer-storage/src/judge.rs
+++ b/foyer-storage/src/judge.rs
@@ -34,9 +34,7 @@ impl Judges {
     }
 
     pub fn with_mask(size: usize, mask: Bitmap<64>) -> Self {
-        let mut umask = mask.bitand(Bitmap::from_value(
-            1u64.wrapping_shl(size as u32).wrapping_sub(1),
-        ));
+        let mut umask = mask.bitand(Bitmap::from_value(1u64.wrapping_shl(size as u32).wrapping_sub(1)));
         umask.invert();
 
         Self {
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index ad9b3d94..ead1232f 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -16,9 +16,8 @@ use std::sync::{LazyLock, OnceLock};
 
 use prometheus::{
     core::{AtomicU64, GenericGauge, GenericGaugeVec},
-    exponential_buckets, opts, register_histogram_vec_with_registry,
-    register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry, Histogram,
-    HistogramVec, IntCounter, IntCounterVec, IntGaugeVec, Registry,
+    exponential_buckets, opts, register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
+    register_int_gauge_vec_with_registry, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGaugeVec, Registry,
 };
 type UintGaugeVec = GenericGaugeVec<AtomicU64>;
 type UintGauge = GenericGauge<AtomicU64>;
@@ -26,9 +25,7 @@ type UintGauge = GenericGauge<AtomicU64>;
 macro_rules! register_gauge_vec {
     ($TYPE:ident, $OPTS:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
         let gauge_vec = $TYPE::new($OPTS, $LABELS_NAMES).unwrap();
-        $REGISTRY
-            .register(Box::new(gauge_vec.clone()))
-            .map(|_| gauge_vec)
+        $REGISTRY.register(Box::new(gauge_vec.clone())).map(|_| gauge_vec)
     }};
 }
 
@@ -189,25 +186,13 @@ pub struct Metrics {
 
 impl Metrics {
     pub fn new(global: &GlobalMetrics, foyer: &str) -> Self {
-        let op_duration_insert_inserted = global
-            .op_duration
-            .with_label_values(&[foyer, "insert", "inserted"]);
-        let op_duration_insert_filtered = global
-            .op_duration
-            .with_label_values(&[foyer, "insert", "filtered"]);
-        let op_duration_insert_dropped = global
-            .op_duration
-            .with_label_values(&[foyer, "insert", "dropped"]);
-        let op_duration_lookup_hit = global
-            .op_duration
-            .with_label_values(&[foyer, "lookup", "hit"]);
-        let op_duration_lookup_miss = global
-            .op_duration
-            .with_label_values(&[foyer, "lookup", "miss"]);
+        let op_duration_insert_inserted = global.op_duration.with_label_values(&[foyer, "insert", "inserted"]);
+        let op_duration_insert_filtered = global.op_duration.with_label_values(&[foyer, "insert", "filtered"]);
+        let op_duration_insert_dropped = global.op_duration.with_label_values(&[foyer, "insert", "dropped"]);
+        let op_duration_lookup_hit = global.op_duration.with_label_values(&[foyer, "lookup", "hit"]);
+        let op_duration_lookup_miss = global.op_duration.with_label_values(&[foyer, "lookup", "miss"]);
         let op_duration_remove = global.op_duration.with_label_values(&[foyer, "remove", ""]);
-        let slow_op_duration_reclaim = global
-            .slow_op_duration
-            .with_label_values(&[foyer, "reclaim", ""]);
+        let slow_op_duration_reclaim = global.slow_op_duration.with_label_values(&[foyer, "reclaim", ""]);
 
         let op_bytes_insert = global.op_bytes.with_label_values(&[foyer, "insert", ""]);
         let op_bytes_lookup = global.op_bytes.with_label_values(&[foyer, "lookup", ""]);
@@ -235,10 +220,7 @@ impl Metrics {
             global
                 .inner_op_duration
                 .with_label_values(&[foyer, "update_catalog", ""]);
-        let inner_op_duration_entry_flush =
-            global
-                .inner_op_duration
-                .with_label_values(&[foyer, "entry_flush", ""]);
+        let inner_op_duration_entry_flush = global.inner_op_duration.with_label_values(&[foyer, "entry_flush", ""]);
         let inner_op_duration_flusher_handle =
             global
                 .inner_op_duration
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index 5620f72c..f87d865b 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -213,9 +213,7 @@ where
         self.metrics
             .op_bytes_reclaim
             .inc_by(region.device().region_size() as u64);
-        self.metrics
-            .total_bytes
-            .sub(region.device().region_size() as u64);
+        self.metrics.total_bytes.sub(region.device().region_size() as u64);
 
         Ok(())
     }
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 45d33481..bc565b10 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -128,9 +128,7 @@ where
     D: Device,
 {
     pub fn new(id: RegionId, device: D) -> Self {
-        let inner = RegionInner {
-            waits: BTreeMap::new(),
-        };
+        let inner = RegionInner { waits: BTreeMap::new() };
         Self {
             id,
             inner: Arc::new(Mutex::new(inner)),
@@ -156,10 +154,7 @@ where
     /// Load region data by view from device.
     #[expect(clippy::type_complexity)]
     #[tracing::instrument(skip(self, view))]
-    pub async fn load(
-        &self,
-        view: RegionView,
-    ) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
+    pub async fn load(&self, view: RegionView) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
         let res = self
             .load_range(view.offset as usize..view.offset as usize + view.len as usize)
             .await;
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 2578d3e8..4d7cedb0 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -106,9 +106,7 @@ where
     }
 
     pub fn eviction_push(&self, region_id: RegionId) {
-        self.eviction
-            .write()
-            .push(self.items[region_id as usize].clone());
+        self.eviction.write().push(self.items[region_id as usize].clone());
     }
 
     pub fn eviction_pop(&self) -> Option<RegionId> {
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 1f9b7e40..8a7f5a3f 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -170,10 +170,7 @@ where
 
     async fn close(&self) -> Result<()> {
         let store = self.store.clone();
-        self.runtime
-            .spawn(async move { store.close().await })
-            .await
-            .unwrap()
+        self.runtime.spawn(async move { store.close().await }).await.unwrap()
     }
 
     fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 03488f8f..ad3401f0 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -69,22 +69,14 @@ pub trait Storage: Send + Sync + Debug + Clone + 'static {
 pub trait StorageExt: Storage {
     #[must_use]
     #[tracing::instrument(skip(self, value))]
-    fn insert(
-        &self,
-        key: Self::Key,
-        value: Self::Value,
-    ) -> impl Future<Output = Result<bool>> + Send {
+    fn insert(&self, key: Self::Key, value: Self::Value) -> impl Future<Output = Result<bool>> + Send {
         let weight = key.serialized_len() + value.serialized_len();
         self.writer(key, weight).finish(value)
     }
 
     #[must_use]
     #[tracing::instrument(skip(self, value))]
-    fn insert_if_not_exists(
-        &self,
-        key: Self::Key,
-        value: Self::Value,
-    ) -> impl Future<Output = Result<bool>> + Send {
+    fn insert_if_not_exists(&self, key: Self::Key, value: Self::Value) -> impl Future<Output = Result<bool>> + Send {
         async move {
             if self.exists(&key)? {
                 return Ok(false);
@@ -101,12 +93,7 @@ pub trait StorageExt: Storage {
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[must_use]
     #[tracing::instrument(skip(self, f))]
-    fn insert_with<F>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
+    fn insert_with<F>(&self, key: Self::Key, f: F, weight: usize) -> impl Future<Output = Result<bool>> + Send
     where
         F: FnOnce() -> anyhow::Result<Self::Value> + Send,
     {
@@ -233,12 +220,8 @@ pub trait AsyncStorageExt: Storage {
         });
     }
 
-    fn insert_if_not_exists_async_with_callback<F, FU>(
-        &self,
-        key: Self::Key,
-        value: Self::Value,
-        f: F,
-    ) where
+    fn insert_if_not_exists_async_with_callback<F, FU>(&self, key: Self::Key, value: Self::Value, f: F)
+    where
         F: FnOnce(Result<bool>) -> FU + Send + 'static,
         FU: Future<Output = ()> + Send + 'static,
     {
@@ -255,11 +238,7 @@ impl<S: Storage> AsyncStorageExt for S {}
 
 pub trait ForceStorageExt: Storage {
     #[tracing::instrument(skip(self, value))]
-    fn insert_force(
-        &self,
-        key: Self::Key,
-        value: Self::Value,
-    ) -> impl Future<Output = Result<bool>> + Send {
+    fn insert_force(&self, key: Self::Key, value: Self::Value) -> impl Future<Output = Result<bool>> + Send {
         let weight = key.serialized_len() + value.serialized_len();
         let mut writer = self.writer(key, weight);
         writer.force();
@@ -273,12 +252,7 @@ pub trait ForceStorageExt: Storage {
     ///
     /// `weight` MUST be equal to `key.serialized_len() + value.serialized_len()`
     #[tracing::instrument(skip(self, f))]
-    fn insert_force_with<F>(
-        &self,
-        key: Self::Key,
-        f: F,
-        weight: usize,
-    ) -> impl Future<Output = Result<bool>> + Send
+    fn insert_force_with<F>(&self, key: Self::Key, f: F, weight: usize) -> impl Future<Output = Result<bool>> + Send
     where
         F: FnOnce() -> anyhow::Result<Self::Value> + Send,
     {
@@ -419,20 +393,11 @@ mod tests {
         assert!(storage.insert(1, vec![b'x'; KB]).await.unwrap());
         assert!(storage.exists(&1).unwrap());
 
-        assert!(!storage
-            .insert_if_not_exists(1, vec![b'x'; KB])
-            .await
-            .unwrap());
-        assert!(storage
-            .insert_if_not_exists(2, vec![b'x'; KB])
-            .await
-            .unwrap());
+        assert!(!storage.insert_if_not_exists(1, vec![b'x'; KB]).await.unwrap());
+        assert!(storage.insert_if_not_exists(2, vec![b'x'; KB]).await.unwrap());
         assert!(storage.exists(&2).unwrap());
 
-        assert!(storage
-            .insert_with(3, || { Ok(vec![b'x'; KB]) }, KB)
-            .await
-            .unwrap());
+        assert!(storage.insert_with(3, || { Ok(vec![b'x'; KB]) }, KB).await.unwrap());
         assert!(storage.exists(&3).unwrap());
 
         assert!(storage
@@ -462,10 +427,7 @@ mod tests {
         assert!(storage.exists(&6).unwrap());
     }
 
-    async fn exists_with_retry(
-        storage: &impl Storage<Key = u64, Value = Vec<u8>>,
-        key: &u64,
-    ) -> bool {
+    async fn exists_with_retry(storage: &impl Storage<Key = u64, Value = Vec<u8>>, key: &u64) -> bool {
         tokio::time::sleep(Duration::from_millis(1)).await;
         for _ in 0..10 {
             if storage.exists(key).unwrap() {
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index a8058ed9..625ca772 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -30,32 +30,23 @@ use crate::{
     storage::{Storage, StorageWriter},
 };
 
-pub type LruFsStore<K, V> =
-    GenericStore<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
+pub type LruFsStore<K, V> = GenericStore<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
 
-pub type LruFsStoreConfig<K, V> =
-    GenericStoreConfig<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>>;
+pub type LruFsStoreConfig<K, V> = GenericStoreConfig<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>>;
 
-pub type LruFsStoreWriter<K, V> =
-    GenericStoreWriter<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
+pub type LruFsStoreWriter<K, V> = GenericStoreWriter<K, V, FsDevice, Lru<RegionEpItemAdapter<LruLink>>, LruLink>;
 
-pub type LfuFsStore<K, V> =
-    GenericStore<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
+pub type LfuFsStore<K, V> = GenericStore<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
 
-pub type LfuFsStoreConfig<K, V> =
-    GenericStoreConfig<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>>;
+pub type LfuFsStoreConfig<K, V> = GenericStoreConfig<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>>;
 
-pub type LfuFsStoreWriter<K, V> =
-    GenericStoreWriter<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
+pub type LfuFsStoreWriter<K, V> = GenericStoreWriter<K, V, FsDevice, Lfu<RegionEpItemAdapter<LfuLink>>, LfuLink>;
 
-pub type FifoFsStore<K, V> =
-    GenericStore<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
+pub type FifoFsStore<K, V> = GenericStore<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
-pub type FifoFsStoreConfig<K, V> =
-    GenericStoreConfig<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
+pub type FifoFsStoreConfig<K, V> = GenericStoreConfig<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
-pub type FifoFsStoreWriter<K, V> =
-    GenericStoreWriter<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
+pub type FifoFsStoreWriter<K, V> = GenericStoreWriter<K, V, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>, FifoLink>;
 
 #[derive(Debug)]
 pub struct NoneStoreWriter<K: Key, V: Value> {
@@ -176,15 +167,9 @@ where
 {
     fn clone(&self) -> Self {
         match self {
-            Self::LruFsStoreConfig { config } => Self::LruFsStoreConfig {
-                config: config.clone(),
-            },
-            Self::LfuFsStoreConfig { config } => Self::LfuFsStoreConfig {
-                config: config.clone(),
-            },
-            Self::FifoFsStoreConfig { config } => Self::FifoFsStoreConfig {
-                config: config.clone(),
-            },
+            Self::LruFsStoreConfig { config } => Self::LruFsStoreConfig { config: config.clone() },
+            Self::LfuFsStoreConfig { config } => Self::LfuFsStoreConfig { config: config.clone() },
+            Self::FifoFsStoreConfig { config } => Self::FifoFsStoreConfig { config: config.clone() },
             Self::NoneStoreConfig => Self::NoneStoreConfig,
         }
     }
@@ -291,18 +276,10 @@ where
 {
     fn clone(&self) -> Self {
         match self {
-            Self::LruFsStore { store } => Self::LruFsStore {
-                store: store.clone(),
-            },
-            Self::LfuFsStore { store } => Self::LfuFsStore {
-                store: store.clone(),
-            },
-            Self::FifoFsStore { store } => Self::FifoFsStore {
-                store: store.clone(),
-            },
-            Self::NoneStore { store } => Self::NoneStore {
-                store: store.clone(),
-            },
+            Self::LruFsStore { store } => Self::LruFsStore { store: store.clone() },
+            Self::LfuFsStore { store } => Self::LfuFsStore { store: store.clone() },
+            Self::FifoFsStore { store } => Self::FifoFsStore { store: store.clone() },
+            Self::NoneStore { store } => Self::NoneStore { store: store.clone() },
         }
     }
 }
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 48f01bbc..640171e1 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -47,10 +47,7 @@ where
 
     for _ in 0..INSERTS as u64 {
         index += 1;
-        store
-            .insert(index, vec![index as u8; 1 * KB])
-            .await
-            .unwrap();
+        store.insert(index, vec![index as u8; 1 * KB]).await.unwrap();
     }
 
     store.close().await.unwrap();
@@ -59,10 +56,7 @@ where
 
     for i in 0..INSERTS as u64 * (LOOPS + 1) as u64 {
         if remains.contains(&i) {
-            assert_eq!(
-                store.lookup(&i).await.unwrap().unwrap(),
-                vec![i as u8; 1 * KB],
-            );
+            assert_eq!(store.lookup(&i).await.unwrap().unwrap(), vec![i as u8; 1 * KB],);
         } else {
             assert!(store.lookup(&i).await.unwrap().is_none());
         }
@@ -80,10 +74,7 @@ where
 
         for i in 0..INSERTS as u64 * (LOOPS + 1) as u64 {
             if remains.contains(&i) {
-                assert_eq!(
-                    store.lookup(&i).await.unwrap().unwrap(),
-                    vec![i as u8; 1 * KB],
-                );
+                assert_eq!(store.lookup(&i).await.unwrap().unwrap(), vec![i as u8; 1 * KB],);
             } else {
                 assert!(store.lookup(&i).await.unwrap().is_none());
             }
@@ -91,10 +82,7 @@ where
 
         for _ in 0..INSERTS as u64 {
             index += 1;
-            store
-                .insert(index, vec![index as u8; 1 * KB])
-                .await
-                .unwrap();
+            store.insert(index, vec![index as u8; 1 * KB]).await.unwrap();
         }
 
         store.close().await.unwrap();
@@ -103,10 +91,7 @@ where
 
         for i in 0..INSERTS as u64 * (LOOPS + 1) as u64 {
             if remains.contains(&i) {
-                assert_eq!(
-                    store.lookup(&i).await.unwrap().unwrap(),
-                    vec![i as u8; 1 * KB],
-                );
+                assert_eq!(store.lookup(&i).await.unwrap().unwrap(), vec![i as u8; 1 * KB],);
             } else {
                 assert!(store.lookup(&i).await.unwrap().is_none());
             }
diff --git a/rustfmt.toml b/rustfmt.toml
index fdb6e4ee..d9905931 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1,4 +1,6 @@
 imports_granularity = "Crate"
 group_imports = "StdExternalCrate"
 tab_spaces = 4
+wrap_comments = true
+max_width = 120
 comment_width = 120
\ No newline at end of file

From fc3789bdfa912f96c34be2f814c85aa2905da8a8 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 6 Mar 2024 12:32:52 +0800
Subject: [PATCH 217/261] chore: update the PR template (#280)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/pull_request_template.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 5d397a96..3d93cffb 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -4,8 +4,8 @@
 
 ## Checklist
 
-- [ ] I have written necessary rustdoc comments
-- [ ] I have added necessary unit tests and integration tests
-- [ ] I have passed `make check` and `make test` or `make all` in my local envirorment.
+- [ ] I have written the necessary rustdoc comments
+- [ ] I have added the necessary unit tests and integration tests
+- [ ] I have passed `make all` (or `make fast` instead if the old tests are not modified) in my local environment.
 
 ## Related issues or PRs (optional)

From 198b8ae3ba3c16db1841290f4c1fc8cfe36cb13f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 6 Mar 2024 16:59:46 +0800
Subject: [PATCH 218/261] test: add asan and lsan for unit tests on CI (#282)

* test: add asan and lsan for unit tests on CI

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update ci cache key prefix

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      | 53 +++++++++++++++++++++++++++---
 .github/workflows/main.yml         | 51 +++++++++++++++++++++++++---
 .github/workflows/pull-request.yml | 51 +++++++++++++++++++++++++---
 3 files changed, 142 insertions(+), 13 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index e320f870..6b448a05 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -5,7 +5,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-12-26
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231227v1
+  CACHE_KEY_SUFFIX: 20240306
 
 jobs:
   misc-check:
@@ -24,7 +24,7 @@ jobs:
           BINARY: yq_linux_amd64
           BUF_VERSION: 1.0.0-rc6
       - name: Install jq
-        uses: dcarbone/install-jq-action@v2.0.2    
+        uses: dcarbone/install-jq-action@v2.0.2
       - name: Check if CI workflows are up-to-date
         run: |
           ./.github/template/generate.sh --check
@@ -112,7 +112,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -142,7 +142,14 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+      - name: Run Unit Tests With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
@@ -152,6 +159,42 @@ jobs:
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+  lsan:
+    name: run with leak saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-lsan
+      - name: Run Unit Tests With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+      - name: Run foyer-storage-bench With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
@@ -172,7 +215,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@nextest
       - name: Run rust clippy check (madsim)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index fb854135..a80dcb10 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-12-26
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231227v1
+  CACHE_KEY_SUFFIX: 20240306
 jobs:
   misc-check:
     name: misc check
@@ -119,7 +119,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -149,7 +149,14 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+      - name: Run Unit Tests With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
@@ -159,6 +166,42 @@ jobs:
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+  lsan:
+    name: run with leak saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-lsan
+      - name: Run Unit Tests With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+      - name: Run foyer-storage-bench With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
@@ -179,7 +222,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@nextest
       - name: Run rust clippy check (madsim)
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index bf9cff0a..75518119 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -12,7 +12,7 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-12-26
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20231227v1
+  CACHE_KEY_SUFFIX: 20240306
 jobs:
   misc-check:
     name: misc check
@@ -118,7 +118,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
       - name: Run foyer-storage-bench with single worker thread and deadlock detection
         env:
           RUST_BACKTRACE: 1
@@ -148,7 +148,14 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+      - name: Run Unit Tests With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
@@ -158,6 +165,42 @@ jobs:
           cargo build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+  lsan:
+    name: run with leak saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-lsan
+      - name: Run Unit Tests With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+      - name: Run foyer-storage-bench With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
@@ -178,7 +221,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@nextest
       - name: Run rust clippy check (madsim)

From 65c1646e81f82ae10aabb0dbbf04d766fc0caeda Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 7 Mar 2024 11:41:54 +0800
Subject: [PATCH 219/261] feat: introduce new in-memory cache abstraction
 (#270)

* feat: introduce foyer-memory Cache abstraction

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove old foyer-memory code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce base handle

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: make cache abstraction use handle trait with base handle

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: impl fifo eviction

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add basic tests

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refactor directory hierarchy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: call access when get

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make hakari happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix dangling ptr after remove

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce RemovableQueue to make FIFO impl easilier.

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use removable queue to simplify fifo impl

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix removable queue grow token updates

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: tiny refactors

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use full type name for trait associated type

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce associate type Context for handle

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce lru eviction policy for foyer-memory

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: add a simple ut and fix bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused generic type

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use rocksdb lru impl instead of cachelib lru

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: add waiters and entry API for request dedup

TODO: add ut for Cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce new in-memory cache abstraction into foyer-memory

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use intrusive dlist to impl fifo

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use intrusive dlist to impl LRU with high-pri pool

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix reinsert bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add ut and fix some bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: expose foyer-memory via foyer crate

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: allow entry future returns context besides

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make lru context clone and copy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make lru context derive Eq and PartialEq

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: impl Future for Entry

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused comments

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* doc: add crate document

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add more uts for cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add ut to cover not reinsert for full

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix typo

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/{queue.rs => async_queue.rs} |   2 +-
 foyer-common/src/lib.rs                       |   3 +-
 foyer-intrusive/src/collections/dlist.rs      |  39 +-
 foyer-memory/Cargo.toml                       |  18 +-
 foyer-memory/src/cache.rs                     | 888 ++++++++++++++++++
 foyer-memory/src/eviction/fifo.rs             | 238 +++++
 foyer-memory/src/eviction/lru.rs              | 462 +++++++++
 foyer-memory/src/eviction/mod.rs              | 108 +++
 foyer-memory/src/eviction/test_utils.rs       |  21 +
 foyer-memory/src/handle.rs                    | 238 +++++
 foyer-memory/src/indexer.rs                   | 120 +++
 foyer-memory/src/lib.rs                       | 336 ++-----
 foyer-storage/src/region_manager.rs           |   2 +-
 foyer-workspace-hack/Cargo.toml               |   3 +-
 foyer/Cargo.toml                              |   1 +
 foyer/src/lib.rs                              |   1 +
 16 files changed, 2195 insertions(+), 285 deletions(-)
 rename foyer-common/src/{queue.rs => async_queue.rs} (99%)
 create mode 100644 foyer-memory/src/cache.rs
 create mode 100644 foyer-memory/src/eviction/fifo.rs
 create mode 100644 foyer-memory/src/eviction/lru.rs
 create mode 100644 foyer-memory/src/eviction/mod.rs
 create mode 100644 foyer-memory/src/eviction/test_utils.rs
 create mode 100644 foyer-memory/src/handle.rs
 create mode 100644 foyer-memory/src/indexer.rs

diff --git a/foyer-common/src/queue.rs b/foyer-common/src/async_queue.rs
similarity index 99%
rename from foyer-common/src/queue.rs
rename to foyer-common/src/async_queue.rs
index 379365f2..6c9017bd 100644
--- a/foyer-common/src/queue.rs
+++ b/foyer-common/src/async_queue.rs
@@ -110,7 +110,7 @@ mod tests {
         task::{Poll, Poll::Pending},
     };
 
-    use crate::queue::AsyncQueue;
+    use crate::async_queue::AsyncQueue;
 
     #[tokio::test]
     async fn test_basic() {
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 88d0173f..7d570a09 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -17,14 +17,15 @@
 #![feature(bound_map)]
 #![feature(associated_type_defaults)]
 #![feature(cfg_match)]
+#![feature(let_chains)]
 #![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 
+pub mod async_queue;
 pub mod batch;
 pub mod bits;
 pub mod code;
 pub mod continuum;
 pub mod erwlock;
-pub mod queue;
 pub mod range;
 pub mod rate;
 pub mod rated_ticket;
diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index ac224170..de3ccb7d 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -164,6 +164,42 @@ where
         self.len() == 0
     }
 
+    /// Get the prev element of the given `link`.
+    ///
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn prev_of_raw(&self, link: NonNull<DlistLink>) -> Option<&<A::Pointer as Pointer>::Item> {
+        link.as_ref().prev().map(|link| self.adapter.link2item(link).as_ref())
+    }
+
+    /// Get the mutable prev element of the given `link`.
+    ///
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn prev_mut_of_raw(&self, link: NonNull<DlistLink>) -> Option<&mut <A::Pointer as Pointer>::Item> {
+        link.as_ref().prev().map(|link| self.adapter.link2item(link).as_mut())
+    }
+
+    /// Get the next element of the given `link`.
+    ///
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn next_of_raw(&self, link: NonNull<DlistLink>) -> Option<&<A::Pointer as Pointer>::Item> {
+        link.as_ref().next().map(|link| self.adapter.link2item(link).as_ref())
+    }
+
+    /// Get the mutable next element of the given `link`.
+    ///
+    /// # Safety
+    ///
+    /// `link` MUST be in this [`Dlist`].
+    pub unsafe fn next_mut_of_raw(&self, link: NonNull<DlistLink>) -> Option<&mut <A::Pointer as Pointer>::Item> {
+        link.as_ref().next().map(|link| self.adapter.link2item(link).as_mut())
+    }
+
     /// Remove an node that holds the given raw link.
     ///
     /// # Safety
@@ -352,7 +388,8 @@ where
                 return None;
             }
 
-            let mut link = self.link.unwrap();
+            debug_assert!(self.is_valid());
+            let mut link = self.link.unwrap_unchecked();
 
             let item = self.dlist.adapter.link2item(link);
             let ptr = A::Pointer::from_ptr(item.as_ptr());
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 88c0b2fe..4179e2e4 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -13,23 +13,23 @@ homepage = "https://github.com/mrcroxx/foyer"
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
-bytes = "1"
-cmsketch = "0.1"
-foyer-common = { path = "../foyer-common" }
-foyer-intrusive = { path = "../foyer-intrusive" }
+ahash = "0.8"
+bitflags = "2"
+crossbeam = "0.8"
+foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+futures = "0.3"
+hashbrown = "0.14"
+itertools = "0.12"
 libc = "0.2"
-memoffset = "0.9"
 parking_lot = "0.12"
-paste = "1.0"
-rand = "0.8.5"
-tracing = "0.1"
-twox-hash = "1"
+tokio = { workspace = true }
 
 [dev-dependencies]
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 hdrhistogram = "7"
+rand = "0.8"
 rand_mt = "4.2.1"
 tempfile = "3"
 
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
new file mode 100644
index 00000000..c5c7370b
--- /dev/null
+++ b/foyer-memory/src/cache.rs
@@ -0,0 +1,888 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    future::Future,
+    hash::BuildHasher,
+    ops::Deref,
+    ptr::NonNull,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
+
+use ahash::RandomState;
+use crossbeam::queue::ArrayQueue;
+use futures::FutureExt;
+use hashbrown::hash_map::{Entry as HashMapEntry, HashMap};
+use itertools::Itertools;
+use parking_lot::Mutex;
+use tokio::{sync::oneshot, task::JoinHandle};
+
+use crate::{
+    eviction::{
+        fifo::{Fifo, FifoHandle},
+        lru::{Lru, LruHandle},
+        Eviction,
+    },
+    handle::Handle,
+    indexer::{HashTableIndexer, Indexer},
+    Key, Value,
+};
+
+#[expect(clippy::type_complexity)]
+struct CacheShard<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    indexer: I,
+    eviction: E,
+
+    capacity: usize,
+    usage: Arc<AtomicUsize>,
+
+    waiters: HashMap<K, Vec<oneshot::Sender<CacheEntry<K, V, H, E, I, S>>>>,
+
+    /// The object pool to avoid frequent handle allocating, shared by all shards.
+    object_pool: Arc<ArrayQueue<Box<H>>>,
+}
+
+impl<K, V, H, E, I, S> CacheShard<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn new(config: &CacheConfig<E, S>, usage: Arc<AtomicUsize>, object_pool: Arc<ArrayQueue<Box<H>>>) -> Self {
+        let indexer = I::new();
+        let eviction = unsafe { E::new(config) };
+        let capacity = config.capacity / config.shards;
+        let waiters = HashMap::default();
+        Self {
+            indexer,
+            eviction,
+            capacity,
+            usage,
+            waiters,
+            object_pool,
+        }
+    }
+
+    /// Insert a new entry into the cache. The handle for the new entry is returned.
+    unsafe fn insert(
+        &mut self,
+        hash: u64,
+        key: K,
+        value: V,
+        charge: usize,
+        context: H::Context,
+        last_reference_entries: &mut Vec<(K, V)>,
+    ) -> NonNull<H> {
+        let mut handle = self.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
+        handle.init(hash, key, value, charge, context);
+        let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
+
+        self.evict(charge, last_reference_entries);
+
+        debug_assert!(!ptr.as_ref().base().is_in_indexer());
+        if let Some(old) = self.indexer.insert(ptr) {
+            debug_assert!(!old.as_ref().base().is_in_indexer());
+            if old.as_ref().base().is_in_eviction() {
+                self.eviction.remove(old);
+            }
+            debug_assert!(!old.as_ref().base().is_in_eviction());
+            // Because the `old` handle is removed from the indexer, it will not be reinserted again.
+            if let Some(entry) = self.try_release_handle(old, false) {
+                last_reference_entries.push(entry);
+            }
+        }
+        self.eviction.push(ptr);
+
+        debug_assert!(ptr.as_ref().base().is_in_indexer());
+        debug_assert!(ptr.as_ref().base().is_in_indexer());
+
+        self.usage.fetch_add(charge, Ordering::Relaxed);
+        ptr.as_mut().base_mut().inc_refs();
+
+        ptr
+    }
+
+    unsafe fn get(&mut self, hash: u64, key: &K) -> Option<NonNull<H>> {
+        let mut ptr = self.indexer.get(hash, key)?;
+        let base = ptr.as_mut().base_mut();
+        debug_assert!(base.is_in_indexer());
+
+        base.inc_refs();
+        self.eviction.access(ptr);
+
+        Some(ptr)
+    }
+
+    /// Remove a key from the cache.
+    ///
+    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
+    unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V)> {
+        let ptr = self.indexer.remove(hash, key)?;
+        if ptr.as_ref().base().is_in_eviction() {
+            self.eviction.remove(ptr);
+        }
+        debug_assert!(!ptr.as_ref().base().is_in_indexer());
+        debug_assert!(!ptr.as_ref().base().is_in_eviction());
+        self.try_release_handle(ptr, false)
+    }
+
+    /// Clear all cache entries.
+    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V)>) {
+        // TODO(MrCroxx): Avoid collecting here?
+        let ptrs = self.indexer.drain().collect_vec();
+        let eptrs = self.eviction.clear();
+
+        // Assert that the handles in the indexer covers the handles in the eviction container.
+        if cfg!(debug_assertions) {
+            use std::{collections::HashSet as StdHashSet, hash::RandomState as StdRandomState};
+            let ptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(ptrs.iter().copied());
+            let eptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(eptrs.iter().copied());
+            debug_assert!((&eptrs - &ptrs).is_empty());
+        }
+
+        // The handles in the indexer covers the handles in the eviction container.
+        // So only the handles drained from the indexer need to be released.
+        for ptr in ptrs {
+            debug_assert!(!ptr.as_ref().base().is_in_indexer());
+            if let Some(entry) = self.try_release_handle(ptr, false) {
+                last_reference_entries.push(entry);
+            }
+        }
+    }
+
+    unsafe fn evict(&mut self, charge: usize, last_reference_entries: &mut Vec<(K, V)>) {
+        while self.usage.load(Ordering::Relaxed) + charge > self.capacity
+            && let Some(evicted) = self.eviction.pop()
+        {
+            let base = evicted.as_ref().base();
+            debug_assert!(base.is_in_indexer());
+            debug_assert!(!base.is_in_eviction());
+            if let Some(entry) = self.try_release_handle(evicted, false) {
+                last_reference_entries.push(entry);
+            }
+        }
+    }
+
+    /// Release a handle used by an external user.
+    ///
+    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
+    unsafe fn try_release_external_handle(&mut self, mut ptr: NonNull<H>) -> Option<(K, V)> {
+        ptr.as_mut().base_mut().dec_refs();
+        self.try_release_handle(ptr, true)
+    }
+
+    /// Try release handle if there is no external reference and no reinsertion is needed.
+    ///
+    /// Return the entry if the handle is released.
+    ///
+    /// Recycle it if possible.
+    unsafe fn try_release_handle(&mut self, mut ptr: NonNull<H>, reinsert: bool) -> Option<(K, V)> {
+        let base = ptr.as_mut().base_mut();
+
+        if base.has_refs() {
+            return None;
+        }
+
+        debug_assert!(base.is_inited());
+        debug_assert!(!base.has_refs());
+
+        // If the entry is not updated or removed from the cache, try to reinsert it or remove it from the indexer and
+        // the eviction container.
+        if base.is_in_indexer() {
+            // The usage is higher than the capacity means most handles are held externally,
+            // the cache shard cannot release enough charges for the new inserted entries.
+            // In this case, the reinsertion should be given up.
+            if reinsert && self.usage.load(Ordering::Relaxed) <= self.capacity {
+                self.eviction.reinsert(ptr);
+                if ptr.as_ref().base().is_in_eviction() {
+                    return None;
+                }
+            }
+
+            // If the entry has not been reinserted, remove it from the indexer and the eviction container (if needed).
+            self.indexer.remove(base.hash(), base.key());
+            if ptr.as_ref().base().is_in_eviction() {
+                self.eviction.remove(ptr);
+            }
+        }
+
+        // Here the handle is neither in the indexer nor in the eviction container.
+        debug_assert!(!base.is_in_indexer());
+        debug_assert!(!base.is_in_eviction());
+        debug_assert!(!base.has_refs());
+
+        self.usage.fetch_sub(base.charge(), Ordering::Relaxed);
+        let entry = base.take();
+
+        let handle = Box::from_raw(ptr.as_ptr());
+        let _ = self.object_pool.push(handle);
+
+        Some(entry)
+    }
+}
+
+impl<K, V, H, E, I, S> Drop for CacheShard<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn drop(&mut self) {
+        unsafe { self.clear(&mut vec![]) }
+    }
+}
+
+pub struct CacheConfig<E, S = RandomState>
+where
+    E: Eviction,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub capacity: usize,
+    pub shards: usize,
+    pub eviction_config: E::Config,
+    pub object_pool_capacity: usize,
+    pub hash_builder: S,
+}
+
+#[expect(clippy::type_complexity)]
+pub enum Entry<K, V, H, E, I, S, ER>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+    ER: std::error::Error,
+{
+    Invalid,
+    Hit(CacheEntry<K, V, H, E, I, S>),
+    Wait(oneshot::Receiver<CacheEntry<K, V, H, E, I, S>>),
+    Miss(JoinHandle<std::result::Result<CacheEntry<K, V, H, E, I, S>, ER>>),
+}
+
+impl<K, V, H, E, I, S, ER> Default for Entry<K, V, H, E, I, S, ER>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+    ER: std::error::Error,
+{
+    fn default() -> Self {
+        Self::Invalid
+    }
+}
+
+impl<K, V, H, E, I, S, ER> Future for Entry<K, V, H, E, I, S, ER>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+    ER: std::error::Error + From<oneshot::error::RecvError>,
+{
+    type Output = std::result::Result<CacheEntry<K, V, H, E, I, S>, ER>;
+
+    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
+        match &mut *self {
+            Self::Invalid => unreachable!(),
+            Self::Hit(_) => std::task::Poll::Ready(Ok(match std::mem::take(&mut *self) {
+                Entry::Hit(entry) => entry,
+                _ => unreachable!(),
+            })),
+            Self::Wait(waiter) => waiter.poll_unpin(cx).map_err(|err| err.into()),
+            Self::Miss(join_handle) => join_handle.poll_unpin(cx).map(|join_result| join_result.unwrap()),
+        }
+    }
+}
+
+#[expect(clippy::type_complexity)]
+pub struct Cache<K, V, H, E, I, S = RandomState>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    shards: Vec<Mutex<CacheShard<K, V, H, E, I, S>>>,
+
+    capacity: usize,
+    usages: Vec<Arc<AtomicUsize>>,
+
+    hash_builder: S,
+}
+
+impl<K, V, H, E, I, S> Cache<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn new(config: CacheConfig<E, S>) -> Self {
+        let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
+        let object_pool = Arc::new(ArrayQueue::new(config.object_pool_capacity));
+        let shards = usages
+            .iter()
+            .map(|usage| CacheShard::new(&config, usage.clone(), object_pool.clone()))
+            .map(Mutex::new)
+            .collect_vec();
+
+        Self {
+            shards,
+            capacity: config.capacity,
+            usages,
+            hash_builder: config.hash_builder,
+        }
+    }
+
+    pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> CacheEntry<K, V, H, E, I, S> {
+        self.insert_with_context(key, value, charge, H::Context::default())
+    }
+
+    pub fn insert_with_context(
+        self: &Arc<Self>,
+        key: K,
+        value: V,
+        charge: usize,
+        context: H::Context,
+    ) -> CacheEntry<K, V, H, E, I, S> {
+        let hash = self.hash_builder.hash_one(&key);
+
+        let mut to_deallocate = vec![];
+
+        let (entry, waiters) = unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            let waiters = shard.waiters.remove(&key);
+            let mut ptr = shard.insert(hash, key, value, charge, context, &mut to_deallocate);
+            if let Some(waiters) = waiters.as_ref() {
+                ptr.as_mut().base_mut().inc_refs_by(waiters.len());
+            }
+            let entry = CacheEntry {
+                cache: self.clone(),
+                ptr,
+            };
+            (entry, waiters)
+        };
+
+        if let Some(waiters) = waiters {
+            for waiter in waiters {
+                let _ = waiter.send(CacheEntry {
+                    cache: self.clone(),
+                    ptr: entry.ptr,
+                });
+            }
+        }
+
+        // Do not deallocate data within the lock section.
+        // TODO: call listener here.
+        drop(to_deallocate);
+
+        entry
+    }
+
+    pub fn remove(&self, key: &K) {
+        let hash = self.hash_builder.hash_one(key);
+
+        let kv = unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            shard.remove(hash, key)
+        };
+
+        // Do not deallocate data within the lock section.
+        // TODO: call listener here.
+        drop(kv);
+    }
+
+    pub fn get(self: &Arc<Self>, key: &K) -> Option<CacheEntry<K, V, H, E, I, S>> {
+        let hash = self.hash_builder.hash_one(key);
+
+        unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            shard.get(hash, key).map(|ptr| CacheEntry {
+                cache: self.clone(),
+                ptr,
+            })
+        }
+    }
+
+    pub fn clear(&self) {
+        let mut to_deallocate = vec![];
+        for shard in self.shards.iter() {
+            let mut shard = shard.lock();
+            unsafe { shard.clear(&mut to_deallocate) };
+        }
+    }
+
+    pub fn capacity(&self) -> usize {
+        self.capacity
+    }
+
+    pub fn usage(&self) -> usize {
+        self.usages.iter().map(|usage| usage.load(Ordering::Relaxed)).sum()
+    }
+
+    unsafe fn try_release_external_handle(&self, ptr: NonNull<H>) {
+        let entry = {
+            let base = ptr.as_ref().base();
+            let mut shard = self.shards[base.hash() as usize % self.shards.len()].lock();
+            shard.try_release_external_handle(ptr)
+        };
+
+        // Do not deallocate data within the lock section.
+        // TODO: call listener here.
+        drop(entry);
+    }
+}
+
+// TODO(MrCroxx): use `hashbrown::HashTable` with `Handle` may relax the `Clone` bound?
+impl<K, V, H, E, I, S> Cache<K, V, H, E, I, S>
+where
+    K: Key + Clone,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> Entry<K, V, H, E, I, S, ER>
+    where
+        F: FnOnce() -> FU,
+        FU: Future<Output = std::result::Result<(V, usize, Option<H::Context>), ER>> + Send + 'static,
+        ER: std::error::Error + Send + 'static,
+    {
+        let hash = self.hash_builder.hash_one(&key);
+
+        unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            if let Some(ptr) = shard.get(hash, &key) {
+                return Entry::Hit(CacheEntry {
+                    cache: self.clone(),
+                    ptr,
+                });
+            }
+            match shard.waiters.entry(key.clone()) {
+                HashMapEntry::Occupied(mut o) => {
+                    let (tx, rx) = oneshot::channel();
+                    o.get_mut().push(tx);
+                    Entry::Wait(rx)
+                }
+                HashMapEntry::Vacant(v) => {
+                    v.insert(vec![]);
+                    let cache = self.clone();
+                    let future = f();
+                    let join = tokio::spawn(async move {
+                        let (value, charge, context) = match future.await {
+                            Ok((value, charge, context)) => (value, charge, context),
+                            Err(e) => {
+                                let mut shard = cache.shards[hash as usize % cache.shards.len()].lock();
+                                shard.waiters.remove(&key);
+                                return Err(e);
+                            }
+                        };
+
+                        let entry = if let Some(context) = context {
+                            cache.insert_with_context(key, value, charge, context)
+                        } else {
+                            cache.insert(key, value, charge)
+                        };
+
+                        Ok(entry)
+                    });
+                    Entry::Miss(join)
+                }
+            }
+        }
+    }
+}
+
+pub struct CacheEntry<K, V, H, E, I, S = RandomState>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    cache: Arc<Cache<K, V, H, E, I, S>>,
+    ptr: NonNull<H>,
+}
+
+impl<K, V, H, E, I, S> CacheEntry<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn key(&self) -> &H::Key {
+        unsafe { self.ptr.as_ref().base().key() }
+    }
+
+    pub fn value(&self) -> &H::Value {
+        unsafe { self.ptr.as_ref().base().value() }
+    }
+
+    pub fn context(&self) -> &H::Context {
+        unsafe { self.ptr.as_ref().base().context() }
+    }
+
+    pub fn charge(&self) -> usize {
+        unsafe { self.ptr.as_ref().base().charge() }
+    }
+
+    pub fn refs(&self) -> usize {
+        unsafe { self.ptr.as_ref().base().refs() }
+    }
+}
+
+impl<K, V, H, E, I, S> Clone for CacheEntry<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn clone(&self) -> Self {
+        let mut ptr = self.ptr;
+
+        unsafe {
+            let base = ptr.as_mut().base_mut();
+            debug_assert!(base.has_refs());
+            base.inc_refs();
+        }
+
+        Self {
+            cache: self.cache.clone(),
+            ptr,
+        }
+    }
+}
+
+impl<K, V, H, E, I, S> Drop for CacheEntry<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn drop(&mut self) {
+        unsafe { self.cache.try_release_external_handle(self.ptr) }
+    }
+}
+
+impl<K, V, H, E, I, S> Deref for CacheEntry<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    type Target = V;
+
+    fn deref(&self) -> &Self::Target {
+        self.value()
+    }
+}
+
+unsafe impl<K, V, H, E, I, S> Send for CacheEntry<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+}
+unsafe impl<K, V, H, E, I, S> Sync for CacheEntry<K, V, H, E, I, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+}
+
+pub type FifoCache<K, V, S = RandomState> =
+    Cache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, S>;
+pub type FifoCacheConfig<K, V, S = RandomState> = CacheConfig<Fifo<K, V>, S>;
+pub type FifoCacheEntry<K, V, S = RandomState> =
+    CacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, S>;
+
+pub type LruCache<K, V, S = RandomState> =
+    Cache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, S>;
+pub type LruCacheConfig<K, V, S = RandomState> = CacheConfig<Lru<K, V>, S>;
+pub type LruCacheEntry<K, V, S = RandomState> =
+    CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, S>;
+
+#[cfg(test)]
+mod tests {
+    use rand::{rngs::SmallRng, RngCore, SeedableRng};
+
+    use super::*;
+    use crate::eviction::{fifo::FifoConfig, lru::LruConfig, test_utils::TestEviction};
+
+    fn is_send_sync_static<T: Send + Sync + 'static>() {}
+
+    #[test]
+    fn test_send_sync_static() {
+        is_send_sync_static::<FifoCache<(), ()>>();
+        is_send_sync_static::<FifoCacheConfig<(), ()>>();
+        is_send_sync_static::<LruCache<(), ()>>();
+        is_send_sync_static::<LruCacheConfig<(), ()>>();
+    }
+
+    #[test]
+    fn test_cache_fuzzy() {
+        const CAPACITY: usize = 256;
+
+        let config = FifoCacheConfig {
+            capacity: CAPACITY,
+            shards: 4,
+            eviction_config: FifoConfig {},
+            object_pool_capacity: 16,
+            hash_builder: RandomState::default(),
+        };
+        let cache = Arc::new(FifoCache::<u64, u64>::new(config));
+
+        let mut rng = SmallRng::seed_from_u64(114514);
+        for _ in 0..100000 {
+            let key = rng.next_u64();
+            if let Some(entry) = cache.get(&key) {
+                assert_eq!(key, *entry);
+                drop(entry);
+                continue;
+            }
+            cache.insert(key, key, 1);
+        }
+        assert_eq!(cache.usage(), CAPACITY);
+    }
+
+    fn fifo(capacity: usize) -> Arc<FifoCache<u64, String>> {
+        let config = FifoCacheConfig {
+            capacity,
+            shards: 1,
+            eviction_config: FifoConfig {},
+            object_pool_capacity: 1,
+            hash_builder: RandomState::default(),
+        };
+        Arc::new(FifoCache::<u64, String>::new(config))
+    }
+
+    fn lru(capacity: usize) -> Arc<LruCache<u64, String>> {
+        let config = LruCacheConfig {
+            capacity,
+            shards: 1,
+            eviction_config: LruConfig {
+                high_priority_pool_ratio: 0.0,
+            },
+            object_pool_capacity: 1,
+            hash_builder: RandomState::default(),
+        };
+        Arc::new(LruCache::<u64, String>::new(config))
+    }
+
+    fn insert_fifo(cache: &Arc<FifoCache<u64, String>>, key: u64, value: &str) -> FifoCacheEntry<u64, String> {
+        cache.insert(key, value.to_string(), value.len())
+    }
+
+    fn insert_lru(cache: &Arc<LruCache<u64, String>>, key: u64, value: &str) -> LruCacheEntry<u64, String> {
+        cache.insert(key, value.to_string(), value.len())
+    }
+
+    #[test]
+    fn test_reference_count() {
+        let cache = fifo(100);
+
+        let refs = |ptr: NonNull<FifoHandle<u64, String>>| unsafe { ptr.as_ref().base().refs() };
+
+        let e1 = insert_fifo(&cache, 42, "the answer to life, the universe, and everything");
+        let ptr = e1.ptr;
+        assert_eq!(refs(ptr), 1);
+
+        let e2 = cache.get(&42).unwrap();
+        assert_eq!(refs(ptr), 2);
+
+        let e3 = e2.clone();
+        assert_eq!(refs(ptr), 3);
+
+        drop(e2);
+        assert_eq!(refs(ptr), 2);
+
+        drop(e3);
+        assert_eq!(refs(ptr), 1);
+
+        drop(e1);
+        assert_eq!(refs(ptr), 0);
+    }
+
+    #[test]
+    fn test_replace() {
+        let cache = fifo(10);
+
+        insert_fifo(&cache, 114, "xx");
+        assert_eq!(cache.usage(), 2);
+
+        insert_fifo(&cache, 514, "QwQ");
+        assert_eq!(cache.usage(), 5);
+
+        insert_fifo(&cache, 114, "(0.0)");
+        assert_eq!(cache.usage(), 8);
+
+        assert_eq!(
+            cache.shards[0].lock().eviction.dump(),
+            vec![(514, "QwQ".to_string()), (114, "(0.0)".to_string())],
+        );
+    }
+
+    #[test]
+    fn test_replace_with_external_refs() {
+        let cache = fifo(10);
+
+        insert_fifo(&cache, 514, "QwQ");
+        insert_fifo(&cache, 114, "(0.0)");
+
+        let e4 = cache.get(&514).unwrap();
+        let e5 = insert_fifo(&cache, 514, "bili");
+
+        assert_eq!(e4.refs(), 1);
+        assert_eq!(e5.refs(), 1);
+
+        // remains: 514 => QwQ (3), 514 => bili (4)
+        // evicted: 114 => (0.0) (5)
+        assert_eq!(cache.usage(), 7);
+
+        assert!(cache.get(&114).is_none());
+        assert_eq!(cache.get(&514).unwrap().value(), "bili");
+        assert_eq!(e4.value(), "QwQ");
+
+        cache.remove(&514);
+        assert_eq!(e5.value(), "bili");
+
+        drop(e5);
+        assert!(cache.get(&514).is_none());
+        assert_eq!(e4.value(), "QwQ");
+
+        assert_eq!(cache.usage(), 3);
+        drop(e4);
+        assert_eq!(cache.usage(), 0);
+    }
+
+    #[test]
+    fn test_reinsert_while_all_referenced_lru() {
+        let cache = lru(10);
+
+        let e1 = insert_lru(&cache, 1, "111");
+        let e2 = insert_lru(&cache, 2, "222");
+        let e3 = insert_lru(&cache, 3, "333");
+        assert_eq!(cache.usage(), 9);
+
+        // No entry will be released because all of them are referenced externally.
+        let e4 = insert_lru(&cache, 4, "444");
+        assert_eq!(cache.usage(), 12);
+
+        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
+        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
+
+        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
+        drop(e1);
+        assert_eq!(cache.usage(), 9);
+
+        // `222` and `333` will be reinserted
+        drop(e2);
+        drop(e3);
+        assert_eq!(
+            cache.shards[0].lock().eviction.dump(),
+            vec![(4, "444".to_string()), (2, "222".to_string()), (3, "333".to_string()),]
+        );
+        assert_eq!(cache.usage(), 9);
+
+        // `444` will be reinserted
+        drop(e4);
+        assert_eq!(
+            cache.shards[0].lock().eviction.dump(),
+            vec![(2, "222".to_string()), (3, "333".to_string()), (4, "444".to_string()),]
+        );
+        assert_eq!(cache.usage(), 9);
+    }
+
+    #[test]
+    fn test_reinsert_while_all_referenced_fifo() {
+        let cache = fifo(10);
+
+        let e1 = insert_fifo(&cache, 1, "111");
+        let e2 = insert_fifo(&cache, 2, "222");
+        let e3 = insert_fifo(&cache, 3, "333");
+        assert_eq!(cache.usage(), 9);
+
+        // No entry will be released because all of them are referenced externally.
+        let e4 = insert_fifo(&cache, 4, "444");
+        assert_eq!(cache.usage(), 12);
+
+        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
+        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
+
+        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
+        drop(e1);
+        assert_eq!(cache.usage(), 9);
+
+        // `222` and `333` will be not reinserted because fifo will ignore reinsert operations.
+        drop([e2, e3, e4]);
+        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
+        assert_eq!(cache.usage(), 3);
+
+        // Note:
+        //
+        // For cache policy like FIFO, the entries will not be reinserted while all handles are referenced.
+        // It's okay for this is not a common situation and is not supposed to happen in real workload.
+    }
+}
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
new file mode 100644
index 00000000..761ad112
--- /dev/null
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -0,0 +1,238 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, hash::BuildHasher, ptr::NonNull};
+
+use foyer_intrusive::{
+    collections::dlist::{Dlist, DlistLink},
+    intrusive_adapter,
+};
+
+use crate::{
+    cache::CacheConfig,
+    eviction::Eviction,
+    handle::{BaseHandle, Handle},
+    Key, Value,
+};
+
+pub type FifoContext = ();
+
+pub struct FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    link: DlistLink,
+    base: BaseHandle<K, V, FifoContext>,
+}
+
+impl<K, V> Debug for FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("FifoHandle").finish()
+    }
+}
+
+intrusive_adapter! { FifoHandleDlistAdapter<K, V> = NonNull<FifoHandle<K, V>>: FifoHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+
+impl<K, V> Handle for FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Context = FifoContext;
+
+    fn new() -> Self {
+        Self {
+            link: DlistLink::default(),
+            base: BaseHandle::new(),
+        }
+    }
+
+    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
+        self.base.init(hash, key, value, charge, context);
+    }
+
+    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &self.base
+    }
+
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &mut self.base
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FifoConfig {}
+
+pub struct Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    queue: Dlist<FifoHandleDlistAdapter<K, V>>,
+}
+
+impl<K, V> Eviction for Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Handle = FifoHandle<K, V>;
+    type Config = FifoConfig;
+
+    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(_: &CacheConfig<Self, S>) -> Self
+    where
+        Self: Sized,
+    {
+        Self { queue: Dlist::new() }
+    }
+
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+        self.queue.push_back(ptr);
+        ptr.as_mut().base_mut().set_in_eviction(true);
+    }
+
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+        self.queue.pop_front().map(|mut ptr| {
+            ptr.as_mut().base_mut().set_in_eviction(false);
+            ptr
+        })
+    }
+
+    unsafe fn reinsert(&mut self, _: NonNull<Self::Handle>) {}
+
+    unsafe fn access(&mut self, _: NonNull<Self::Handle>) {}
+
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let p = self.queue.iter_mut_from_raw(ptr.as_mut().link.raw()).remove().unwrap();
+        assert_eq!(p, ptr);
+        ptr.as_mut().base_mut().set_in_eviction(false);
+    }
+
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+        let mut res = Vec::with_capacity(self.len());
+        while let Some(mut ptr) = self.queue.pop_front() {
+            ptr.as_mut().base_mut().set_in_eviction(false);
+            res.push(ptr);
+        }
+        res
+    }
+
+    unsafe fn len(&self) -> usize {
+        self.queue.len()
+    }
+
+    unsafe fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+unsafe impl<K, V> Send for Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+unsafe impl<K, V> Sync for Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+
+#[cfg(test)]
+pub mod tests {
+    use ahash::RandomState;
+    use itertools::Itertools;
+
+    use super::*;
+    use crate::eviction::test_utils::TestEviction;
+
+    impl<K, V> TestEviction for Fifo<K, V>
+    where
+        K: Key + Clone,
+        V: Value + Clone,
+    {
+        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+            self.queue
+                .iter()
+                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .collect_vec()
+        }
+    }
+
+    type TestFifoHandle = FifoHandle<u64, u64>;
+    type TestFifo = Fifo<u64, u64>;
+
+    unsafe fn new_test_fifo_handle_ptr(key: u64, value: u64) -> NonNull<TestFifoHandle> {
+        let mut handle = Box::new(TestFifoHandle::new());
+        handle.init(0, key, value, 1, ());
+        NonNull::new_unchecked(Box::into_raw(handle))
+    }
+
+    unsafe fn del_test_fifo_handle_ptr(ptr: NonNull<TestFifoHandle>) {
+        let _ = Box::from_raw(ptr.as_ptr());
+    }
+
+    #[test]
+    fn test_fifo() {
+        unsafe {
+            let ptrs = (0..8).map(|i| new_test_fifo_handle_ptr(i, i)).collect_vec();
+
+            let config = CacheConfig {
+                capacity: 0,
+                shards: 1,
+                eviction_config: FifoConfig {},
+                object_pool_capacity: 0,
+                hash_builder: RandomState::default(),
+            };
+
+            let mut fifo = TestFifo::new(&config);
+
+            // 0, 1, 2, 3
+            fifo.push(ptrs[0]);
+            fifo.push(ptrs[1]);
+            fifo.push(ptrs[2]);
+            fifo.push(ptrs[3]);
+
+            // 2, 3
+            let p0 = fifo.pop().unwrap();
+            let p1 = fifo.pop().unwrap();
+            assert_eq!(ptrs[0], p0);
+            assert_eq!(ptrs[1], p1);
+
+            // 2, 3, 4, 5, 6
+            fifo.push(ptrs[4]);
+            fifo.push(ptrs[5]);
+            fifo.push(ptrs[6]);
+
+            // 2, 6
+            fifo.remove(ptrs[3]);
+            fifo.remove(ptrs[4]);
+            fifo.remove(ptrs[5]);
+
+            assert_eq!(fifo.clear(), vec![ptrs[2], ptrs[6]]);
+
+            for ptr in ptrs {
+                del_test_fifo_handle_ptr(ptr);
+            }
+        }
+    }
+}
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
new file mode 100644
index 00000000..921e2abe
--- /dev/null
+++ b/foyer-memory/src/eviction/lru.rs
@@ -0,0 +1,462 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, hash::BuildHasher, ptr::NonNull};
+
+use foyer_intrusive::{
+    collections::dlist::{Dlist, DlistLink},
+    core::adapter::Link,
+    intrusive_adapter,
+};
+
+use crate::{
+    cache::CacheConfig,
+    eviction::Eviction,
+    handle::{BaseHandle, Handle},
+    Key, Value,
+};
+
+#[derive(Debug)]
+pub struct LruConfig {
+    /// The ratio of the high priority pool occupied.
+    ///
+    /// [`Lru`] guarantees that the high priority charges are always as larger as
+    /// but no larger that the capacity * high priority pool ratio.
+    ///
+    /// # Panic
+    ///
+    /// Panics if the value is not in [0, 1.0].
+    pub high_priority_pool_ratio: f64,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LruContext {
+    HighPriority,
+    LowPriority,
+}
+
+impl Default for LruContext {
+    fn default() -> Self {
+        Self::HighPriority
+    }
+}
+
+pub struct LruHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    link: DlistLink,
+    base: BaseHandle<K, V, LruContext>,
+    in_high_priority_pool: bool,
+}
+
+impl<K, V> Debug for LruHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LruHandle").finish()
+    }
+}
+
+intrusive_adapter! { LruHandleDlistAdapter<K, V> = NonNull<LruHandle<K, V>>: LruHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+
+impl<K, V> Handle for LruHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Context = LruContext;
+
+    fn new() -> Self {
+        Self {
+            link: DlistLink::default(),
+            base: BaseHandle::new(),
+            in_high_priority_pool: false,
+        }
+    }
+
+    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
+        self.base.init(hash, key, value, charge, context)
+    }
+
+    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &self.base
+    }
+
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &mut self.base
+    }
+}
+
+unsafe impl<K, V> Send for LruHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+unsafe impl<K, V> Sync for LruHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+
+pub struct Lru<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    high_priority_list: Dlist<LruHandleDlistAdapter<K, V>>,
+    list: Dlist<LruHandleDlistAdapter<K, V>>,
+
+    high_priority_charges: usize,
+    high_priority_charges_capacity: usize,
+}
+
+impl<K, V> Lru<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    unsafe fn may_overflow_high_priority_pool(&mut self) {
+        while self.high_priority_charges > self.high_priority_charges_capacity {
+            debug_assert!(!self.high_priority_list.is_empty());
+
+            // overflow last entry in high priority pool to low priority pool
+            let mut ptr = self.high_priority_list.pop_front().unwrap_unchecked();
+            ptr.as_mut().in_high_priority_pool = false;
+            self.high_priority_charges -= ptr.as_ref().base().charge();
+            self.list.push_back(ptr);
+        }
+    }
+}
+
+impl<K, V> Eviction for Lru<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Handle = LruHandle<K, V>;
+    type Config = LruConfig;
+
+    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(config: &CacheConfig<Self, S>) -> Self
+    where
+        Self: Sized,
+    {
+        assert!(
+            config.eviction_config.high_priority_pool_ratio >= 0.0
+                && config.eviction_config.high_priority_pool_ratio <= 1.0,
+            "high_priority_pool_ratio_percentage must be in [0, 100], given: {}",
+            config.eviction_config.high_priority_pool_ratio
+        );
+
+        let high_priority_charges_capacity =
+            config.capacity as f64 * config.eviction_config.high_priority_pool_ratio / config.shards as f64;
+        let high_priority_charges_capacity = high_priority_charges_capacity as usize;
+
+        Self {
+            high_priority_list: Dlist::new(),
+            list: Dlist::new(),
+            high_priority_charges: 0,
+            high_priority_charges_capacity,
+        }
+    }
+
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+
+        debug_assert!(!handle.link.is_linked());
+
+        match handle.base().context() {
+            LruContext::HighPriority => {
+                handle.in_high_priority_pool = true;
+                self.high_priority_charges += handle.base().charge();
+                self.high_priority_list.push_back(ptr);
+
+                self.may_overflow_high_priority_pool();
+            }
+            LruContext::LowPriority => {
+                handle.in_high_priority_pool = false;
+                self.list.push_back(ptr);
+            }
+        }
+
+        handle.base_mut().set_in_eviction(true);
+    }
+
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+        let mut ptr = self.list.pop_front().or_else(|| self.high_priority_list.pop_front())?;
+
+        let handle = ptr.as_mut();
+        debug_assert!(!handle.link.is_linked());
+
+        if handle.in_high_priority_pool {
+            self.high_priority_charges -= handle.base().charge();
+        }
+
+        handle.base_mut().set_in_eviction(false);
+
+        Some(ptr)
+    }
+
+    unsafe fn access(&mut self, _: NonNull<Self::Handle>) {}
+
+    unsafe fn reinsert(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+
+        if handle.base().is_in_eviction() {
+            debug_assert!(handle.link.is_linked());
+            self.remove(ptr);
+            self.push(ptr);
+        } else {
+            debug_assert!(!handle.link.is_linked());
+            self.push(ptr);
+        }
+    }
+
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+        debug_assert!(handle.link.is_linked());
+
+        if handle.in_high_priority_pool {
+            self.high_priority_charges -= handle.base.charge();
+            self.high_priority_list.remove_raw(handle.link.raw());
+        } else {
+            self.list.remove_raw(handle.link.raw());
+        }
+
+        handle.base_mut().set_in_eviction(false);
+    }
+
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+        let mut res = Vec::with_capacity(self.len());
+
+        while !self.list.is_empty() {
+            let mut ptr = self.list.pop_front().unwrap_unchecked();
+            ptr.as_mut().base_mut().set_in_eviction(false);
+            res.push(ptr);
+        }
+
+        while !self.high_priority_list.is_empty() {
+            let mut ptr = self.high_priority_list.pop_front().unwrap_unchecked();
+            ptr.as_mut().base_mut().set_in_eviction(false);
+            self.high_priority_charges -= ptr.as_ref().base().charge();
+            res.push(ptr);
+        }
+
+        debug_assert_eq!(self.high_priority_charges, 0);
+
+        res
+    }
+
+    unsafe fn len(&self) -> usize {
+        self.high_priority_list.len() + self.list.len()
+    }
+
+    unsafe fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+unsafe impl<K, V> Send for Lru<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+unsafe impl<K, V> Sync for Lru<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+
+#[cfg(test)]
+pub mod tests {
+    use ahash::RandomState;
+    use foyer_intrusive::core::pointer::Pointer;
+    use itertools::Itertools;
+
+    use super::*;
+    use crate::eviction::test_utils::TestEviction;
+
+    impl<K, V> TestEviction for Lru<K, V>
+    where
+        K: Key + Clone,
+        V: Value + Clone,
+    {
+        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+            self.list
+                .iter()
+                .chain(self.high_priority_list.iter())
+                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .collect_vec()
+        }
+    }
+
+    type TestLruHandle = LruHandle<u64, u64>;
+    type TestLru = Lru<u64, u64>;
+
+    unsafe fn new_test_lru_handle_ptr(key: u64, value: u64, context: LruContext) -> NonNull<TestLruHandle> {
+        let mut handle = Box::new(TestLruHandle::new());
+        handle.init(0, key, value, 1, context);
+        NonNull::new_unchecked(Box::into_raw(handle))
+    }
+
+    unsafe fn del_test_lru_handle_ptr(ptr: NonNull<TestLruHandle>) {
+        let _ = Box::from_raw(ptr.as_ptr());
+    }
+
+    unsafe fn dump_test_lru(lru: &TestLru) -> (Vec<NonNull<TestLruHandle>>, Vec<NonNull<TestLruHandle>>) {
+        (
+            lru.list
+                .iter()
+                .map(|handle| NonNull::new_unchecked(handle.as_ptr() as *mut _))
+                .collect_vec(),
+            lru.high_priority_list
+                .iter()
+                .map(|handle| NonNull::new_unchecked(handle.as_ptr() as *mut _))
+                .collect_vec(),
+        )
+    }
+
+    #[test]
+    fn test_lru() {
+        unsafe {
+            let ptrs = (0..20)
+                .map(|i| {
+                    new_test_lru_handle_ptr(
+                        i,
+                        i,
+                        if i < 10 {
+                            LruContext::HighPriority
+                        } else {
+                            LruContext::LowPriority
+                        },
+                    )
+                })
+                .collect_vec();
+
+            let config = CacheConfig {
+                capacity: 8,
+                shards: 1,
+                eviction_config: LruConfig {
+                    high_priority_pool_ratio: 0.5,
+                },
+                object_pool_capacity: 0,
+                hash_builder: RandomState::default(),
+            };
+            let mut lru = TestLru::new(&config);
+
+            assert_eq!(lru.high_priority_charges_capacity, 4);
+
+            // [0, 1, 2, 3]
+            lru.push(ptrs[0]);
+            lru.push(ptrs[1]);
+            lru.push(ptrs[2]);
+            lru.push(ptrs[3]);
+            assert_eq!(lru.len(), 4);
+            assert_eq!(lru.high_priority_charges, 4);
+            assert_eq!(lru.high_priority_list.len(), 4);
+            assert_eq!(dump_test_lru(&lru), (vec![], vec![ptrs[0], ptrs[1], ptrs[2], ptrs[3]]));
+
+            // 0, [1, 2, 3, 4]
+            lru.push(ptrs[4]);
+            assert_eq!(lru.len(), 5);
+            assert_eq!(lru.high_priority_charges, 4);
+            assert_eq!(lru.high_priority_list.len(), 4);
+            assert_eq!(
+                dump_test_lru(&lru),
+                (vec![ptrs[0]], vec![ptrs[1], ptrs[2], ptrs[3], ptrs[4]])
+            );
+
+            // 0, 10, [1, 2, 3, 4]
+            lru.push(ptrs[10]);
+            assert_eq!(lru.len(), 6);
+            assert_eq!(lru.high_priority_charges, 4);
+            assert_eq!(lru.high_priority_list.len(), 4);
+            assert_eq!(
+                dump_test_lru(&lru),
+                (vec![ptrs[0], ptrs[10]], vec![ptrs[1], ptrs[2], ptrs[3], ptrs[4]])
+            );
+
+            // 10, [1, 2, 3, 4]
+            let p0 = lru.pop().unwrap();
+            assert_eq!(ptrs[0], p0);
+            assert_eq!(lru.len(), 5);
+            assert_eq!(lru.high_priority_charges, 4);
+            assert_eq!(lru.high_priority_list.len(), 4);
+            assert_eq!(
+                dump_test_lru(&lru),
+                (vec![ptrs[10]], vec![ptrs[1], ptrs[2], ptrs[3], ptrs[4]])
+            );
+
+            // 10, [1, 3, 4]
+            lru.remove(ptrs[2]);
+            assert_eq!(lru.len(), 4);
+            assert_eq!(lru.high_priority_charges, 3);
+            assert_eq!(lru.high_priority_list.len(), 3);
+            assert_eq!(dump_test_lru(&lru), (vec![ptrs[10]], vec![ptrs[1], ptrs[3], ptrs[4]]));
+
+            // 10, 11, [1, 3, 4]
+            lru.push(ptrs[11]);
+            assert_eq!(lru.len(), 5);
+            assert_eq!(lru.high_priority_charges, 3);
+            assert_eq!(lru.high_priority_list.len(), 3);
+            assert_eq!(
+                dump_test_lru(&lru),
+                (vec![ptrs[10], ptrs[11]], vec![ptrs[1], ptrs[3], ptrs[4]])
+            );
+
+            // 10, 11, 1, [3, 4, 5, 6]
+            lru.push(ptrs[5]);
+            lru.push(ptrs[6]);
+            assert_eq!(lru.len(), 7);
+            assert_eq!(lru.high_priority_charges, 4);
+            assert_eq!(lru.high_priority_list.len(), 4);
+            assert_eq!(
+                dump_test_lru(&lru),
+                (
+                    vec![ptrs[10], ptrs[11], ptrs[1]],
+                    vec![ptrs[3], ptrs[4], ptrs[5], ptrs[6]]
+                )
+            );
+
+            // 10, 11, 1, 3, [4, 5, 6, 0]
+            lru.push(ptrs[0]);
+            assert_eq!(lru.len(), 8);
+            assert_eq!(lru.high_priority_charges, 4);
+            assert_eq!(lru.high_priority_list.len(), 4);
+            assert_eq!(
+                dump_test_lru(&lru),
+                (
+                    vec![ptrs[10], ptrs[11], ptrs[1], ptrs[3]],
+                    vec![ptrs[4], ptrs[5], ptrs[6], ptrs[0]]
+                )
+            );
+
+            let ps = lru.clear();
+            assert_eq!(ps, [10, 11, 1, 3, 4, 5, 6, 0].map(|i| ptrs[i]));
+
+            for ptr in ptrs {
+                del_test_lru_handle_ptr(ptr);
+            }
+        }
+    }
+}
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
new file mode 100644
index 00000000..98846d2f
--- /dev/null
+++ b/foyer-memory/src/eviction/mod.rs
@@ -0,0 +1,108 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{hash::BuildHasher, ptr::NonNull};
+
+use crate::{cache::CacheConfig, handle::Handle};
+
+/// The lifetime of `handle: Self::H` is managed by [`Indexer`].
+///
+/// Each `handle`'s lifetime in [`Indexer`] must outlive the raw pointer in [`Eviction`].
+pub trait Eviction: Send + Sync + 'static {
+    type Handle: Handle;
+    type Config;
+
+    /// Create a new empty eviction container.
+    ///
+    /// # Safety
+    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(config: &CacheConfig<Self, S>) -> Self
+    where
+        Self: Sized;
+
+    /// Push a handle `ptr` into the eviction container.
+    ///
+    /// The caller guarantees that the `ptr` is NOT in the eviction container.
+    ///
+    /// # Safety
+    ///
+    /// The `ptr` must be kept holding until `pop` or `remove`.
+    ///
+    /// The base handle associated to the `ptr` must be set in cache.
+    unsafe fn push(&mut self, ptr: NonNull<Self::Handle>);
+
+    /// Pop a handle `ptr` from the eviction container.
+    ///
+    /// # Safety
+    ///
+    /// The `ptr` must be taken from the eviction container.
+    /// Or it may become dangling and cause UB.
+    ///
+    /// The base handle associated to the `ptr` must be set NOT in cache.
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>>;
+
+    /// Try to reinsert a handle `ptr` into the eviction container after access.
+    ///
+    /// The eviction container can decide if to insert based on the algorithm.
+    ///
+    /// # Safety
+    ///
+    /// The given `ptr` may be either IN or NOT IN the eviction container.
+    /// If the `ptr` is reinserted, the base handle associated to it must be set in cache.
+    /// If the `ptr` is NOT reinserted, the base handle associated to it must be set NOT in cache.
+    unsafe fn reinsert(&mut self, ptr: NonNull<Self::Handle>);
+
+    /// Notify the eviciton container that the `ptr` is accessed.
+    /// The eviction container can update its statistics.
+    ///
+    /// # Safety
+    ///
+    /// The given `ptr` can be EITHER in the eviction container OR not in the eviction container.
+    unsafe fn access(&mut self, ptr: NonNull<Self::Handle>);
+
+    /// Remove the given `ptr` from the eviction container.
+    ///
+    /// /// The caller guarantees that the `ptr` is NOT in the eviction container.
+    ///
+    /// # Safety
+    ///
+    /// The `ptr` must be taken from the eviction container, otherwise it may become dangling and cause UB.
+    ///
+    /// The base handle associated to the `ptr` must be set NOT in cache.
+    unsafe fn remove(&mut self, ptr: NonNull<Self::Handle>);
+
+    /// Remove all `ptr`s from the eviction container and reset.
+    ///
+    /// # Safety
+    ///
+    /// All `ptr` must be taken from the eviction container, otherwise it may become dangling and cause UB.
+    ///
+    /// All base handles associated to the `ptr`s must be set NOT in cache.
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>>;
+
+    /// Return the count of the `ptr`s that in the eviction container.
+    ///
+    /// # Safety
+    unsafe fn len(&self) -> usize;
+
+    /// Return `true` if the eviction container is empty.
+    ///
+    /// # Safety
+    unsafe fn is_empty(&self) -> bool;
+}
+
+pub mod fifo;
+pub mod lru;
+
+#[cfg(test)]
+pub mod test_utils;
diff --git a/foyer-memory/src/eviction/test_utils.rs b/foyer-memory/src/eviction/test_utils.rs
new file mode 100644
index 00000000..e266d405
--- /dev/null
+++ b/foyer-memory/src/eviction/test_utils.rs
@@ -0,0 +1,21 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use super::Eviction;
+use crate::handle::Handle;
+
+#[expect(clippy::type_complexity)]
+pub trait TestEviction: Eviction {
+    fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)>;
+}
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
new file mode 100644
index 00000000..e1be5cdb
--- /dev/null
+++ b/foyer-memory/src/handle.rs
@@ -0,0 +1,238 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use bitflags::bitflags;
+
+use crate::{Key, Value};
+
+bitflags! {
+    #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+    struct BaseHandleFlags: u8 {
+        const IN_INDEXER = 0b00000001;
+        const IN_EVICTION = 0b00000010;
+    }
+}
+
+pub trait Handle: Send + Sync + 'static {
+    type Key: Key;
+    type Value: Value;
+    type Context: Default;
+
+    fn new() -> Self;
+    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context);
+
+    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context>;
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context>;
+}
+
+#[derive(Debug)]
+pub struct BaseHandle<K, V, C>
+where
+    K: Key,
+    V: Value,
+{
+    /// key, value, context
+    entry: Option<(K, V, C)>,
+    /// key hash
+    hash: u64,
+    /// entry charge
+    charge: usize,
+    /// external reference count
+    refs: usize,
+    /// flags that used by the general cache abstraction
+    flags: BaseHandleFlags,
+}
+
+impl<K, V, C> Default for BaseHandle<K, V, C>
+where
+    K: Key,
+    V: Value,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<K, V, C> BaseHandle<K, V, C>
+where
+    K: Key,
+    V: Value,
+{
+    /// Create a uninited handle.
+    #[inline(always)]
+    pub fn new() -> Self {
+        Self {
+            entry: None,
+            hash: 0,
+            charge: 0,
+            refs: 0,
+            flags: BaseHandleFlags::empty(),
+        }
+    }
+
+    /// Init handle with args.
+    #[inline(always)]
+    pub fn init(&mut self, hash: u64, key: K, value: V, charge: usize, context: C) {
+        debug_assert!(self.entry.is_none());
+        self.hash = hash;
+        self.entry = Some((key, value, context));
+        self.charge = charge;
+        self.refs = 0;
+        self.flags = BaseHandleFlags::empty();
+    }
+
+    /// Take key and value from the handle and reset it to the uninited state.
+    #[inline(always)]
+    pub fn take(&mut self) -> (K, V) {
+        debug_assert!(self.entry.is_some());
+        unsafe { self.entry.take().map(|(key, value, _)| (key, value)).unwrap_unchecked() }
+    }
+
+    /// Return `true` if the handle is inited.
+    #[inline(always)]
+    pub fn is_inited(&self) -> bool {
+        self.entry.is_some()
+    }
+
+    /// Get key hash.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the handle is uninited.
+    #[inline(always)]
+    pub fn hash(&self) -> u64 {
+        self.hash
+    }
+
+    /// Get key reference.
+    ///  
+    /// # Panics
+    ///
+    /// Panics if the handle is uninited.
+    #[inline(always)]
+    pub fn key(&self) -> &K {
+        debug_assert!(self.entry.is_some());
+        unsafe { self.entry.as_ref().map(|entry| &entry.0).unwrap_unchecked() }
+    }
+
+    /// Get value reference.
+    ///  
+    /// # Panics
+    ///
+    /// Panics if the handle is uninited.
+    #[inline(always)]
+    pub fn value(&self) -> &V {
+        debug_assert!(self.entry.is_some());
+        unsafe { self.entry.as_ref().map(|entry| &entry.1).unwrap_unchecked() }
+    }
+
+    /// Get context reference.
+    ///  
+    /// # Panics
+    ///
+    /// Panics if the handle is uninited.
+    #[inline(always)]
+    pub fn context(&self) -> &C {
+        debug_assert!(self.entry.is_some());
+        unsafe { self.entry.as_ref().map(|entry| &entry.2).unwrap_unchecked() }
+    }
+
+    /// Get the charge of the handle.
+    #[inline(always)]
+    pub fn charge(&self) -> usize {
+        self.charge
+    }
+
+    /// Increase the external reference count of the handle, returns the new reference count.
+    #[inline(always)]
+    pub fn inc_refs(&mut self) -> usize {
+        self.inc_refs_by(1)
+    }
+
+    /// Increase the external reference count of the handle, returns the new reference count.
+    #[inline(always)]
+    pub fn inc_refs_by(&mut self, val: usize) -> usize {
+        self.refs += val;
+        self.refs
+    }
+
+    /// Decrease the external reference count of the handle, returns the new reference count.
+    #[inline(always)]
+    pub fn dec_refs(&mut self) -> usize {
+        self.refs -= 1;
+        self.refs
+    }
+
+    /// Get the external reference count of the handle.
+    #[inline(always)]
+    pub fn refs(&self) -> usize {
+        self.refs
+    }
+
+    /// Return `true` if there are external references.
+    #[inline(always)]
+    pub fn has_refs(&self) -> bool {
+        self.refs() > 0
+    }
+
+    #[inline(always)]
+    pub fn set_in_indexer(&mut self, in_cache: bool) {
+        if in_cache {
+            self.flags |= BaseHandleFlags::IN_INDEXER;
+        } else {
+            self.flags -= BaseHandleFlags::IN_INDEXER;
+        }
+    }
+
+    #[inline(always)]
+    pub fn is_in_indexer(&self) -> bool {
+        !(self.flags & BaseHandleFlags::IN_INDEXER).is_empty()
+    }
+
+    #[inline(always)]
+    pub fn set_in_eviction(&mut self, in_eviction: bool) {
+        if in_eviction {
+            self.flags |= BaseHandleFlags::IN_EVICTION;
+        } else {
+            self.flags -= BaseHandleFlags::IN_EVICTION;
+        }
+    }
+
+    #[inline(always)]
+    pub fn is_in_eviction(&self) -> bool {
+        !(self.flags & BaseHandleFlags::IN_EVICTION).is_empty()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_base_handle_basic() {
+        let mut h = BaseHandle::<(), (), ()>::new();
+        assert!(!h.is_in_indexer());
+        assert!(!h.is_in_eviction());
+
+        h.set_in_indexer(true);
+        h.set_in_eviction(true);
+        assert!(h.is_in_indexer());
+        assert!(h.is_in_eviction());
+
+        h.set_in_indexer(false);
+        h.set_in_eviction(false);
+        assert!(!h.is_in_indexer());
+        assert!(!h.is_in_eviction());
+    }
+}
diff --git a/foyer-memory/src/indexer.rs b/foyer-memory/src/indexer.rs
new file mode 100644
index 00000000..fd49557c
--- /dev/null
+++ b/foyer-memory/src/indexer.rs
@@ -0,0 +1,120 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::ptr::NonNull;
+
+use hashbrown::hash_table::{Entry as HashTableEntry, HashTable};
+
+use crate::{handle::Handle, Key};
+
+#[expect(clippy::missing_safety_doc)]
+pub trait Indexer: Send + Sync + 'static {
+    type Key: Key;
+    type Handle: Handle<Key = Self::Key>;
+
+    fn new() -> Self;
+    unsafe fn insert(&mut self, handle: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>>;
+    unsafe fn get(&self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>>;
+    unsafe fn remove(&mut self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>>;
+    unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>>;
+}
+
+pub struct HashTableIndexer<K, H>
+where
+    K: Key,
+    H: Handle<Key = K>,
+{
+    table: HashTable<NonNull<H>>,
+}
+
+unsafe impl<K, H> Send for HashTableIndexer<K, H>
+where
+    K: Key,
+    H: Handle<Key = K>,
+{
+}
+
+unsafe impl<K, H> Sync for HashTableIndexer<K, H>
+where
+    K: Key,
+    H: Handle<Key = K>,
+{
+}
+
+impl<K, H> Indexer for HashTableIndexer<K, H>
+where
+    K: Key,
+    H: Handle<Key = K>,
+{
+    type Key = K;
+    type Handle = H;
+
+    fn new() -> Self {
+        Self {
+            table: HashTable::new(),
+        }
+    }
+
+    unsafe fn insert(&mut self, mut ptr: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>> {
+        let base = ptr.as_mut().base_mut();
+
+        debug_assert!(!base.is_in_indexer());
+        base.set_in_indexer(true);
+
+        match self.table.entry(
+            base.hash(),
+            |p| p.as_ref().base().key() == base.key(),
+            |p| p.as_ref().base().hash(),
+        ) {
+            HashTableEntry::Occupied(mut o) => {
+                std::mem::swap(o.get_mut(), &mut ptr);
+                let b = ptr.as_mut().base_mut();
+                debug_assert!(b.is_in_indexer());
+                b.set_in_indexer(false);
+                Some(ptr)
+            }
+            HashTableEntry::Vacant(v) => {
+                v.insert(ptr);
+                None
+            }
+        }
+    }
+
+    unsafe fn get(&self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>> {
+        self.table.find(hash, |p| p.as_ref().base().key() == key).copied()
+    }
+
+    unsafe fn remove(&mut self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>> {
+        match self
+            .table
+            .entry(hash, |p| p.as_ref().base().key() == key, |p| p.as_ref().base().hash())
+        {
+            HashTableEntry::Occupied(o) => {
+                let (mut p, _) = o.remove();
+                let b = p.as_mut().base_mut();
+                debug_assert!(b.is_in_indexer());
+                b.set_in_indexer(false);
+                Some(p)
+            }
+            HashTableEntry::Vacant(_) => None,
+        }
+    }
+
+    unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>> {
+        self.table.drain().map(|mut ptr| {
+            ptr.as_mut().base_mut().set_in_indexer(false);
+            ptr
+        })
+    }
+}
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 0e0dce70..8ff8a433 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -12,274 +12,68 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{hash::Hasher, sync::Arc};
-
-use foyer_common::code::{Key, Value};
-use foyer_intrusive::{
-    collections::hashmap::{HashMap, HashMapLink},
-    core::adapter::Link,
-    eviction::EvictionPolicy,
-    intrusive_adapter, key_adapter, priority_adapter,
-};
-use parking_lot::Mutex;
-use twox_hash::XxHash64;
-
-pub struct CacheConfig<E>
-where
-    E: EvictionPolicy,
-{
-    capacity: usize,
-    shard_bits: usize,
-    hashmap_bits: usize,
-    eviction_config: E::Config,
-}
-
-pub struct Cache<K, V, E, EL>
-where
-    K: Key,
-    V: Value,
-    E: EvictionPolicy<Adapter = CacheItemEpAdapter<K, V, EL>>,
-    EL: Link,
-{
-    shards: Vec<Mutex<CacheShard<K, V, E, EL>>>,
-}
-
-struct CacheShard<K, V, E, EL>
-where
-    K: Key,
-    V: Value,
-    E: EvictionPolicy<Adapter = CacheItemEpAdapter<K, V, EL>>,
-    EL: Link,
-{
-    container: HashMap<K, V, CacheItemHmAdapter<K, V, EL>>,
-    eviction: E,
-
-    capacity: usize,
-    size: usize,
-}
-
-#[derive(Debug)]
-pub struct CacheItem<K, V, EL>
-where
-    K: Key,
-    V: Value,
-    EL: Link,
-{
-    elink: EL,
-    clink: HashMapLink,
-
-    key: K,
-    value: V,
-
-    priority: usize,
-}
-
-impl<K, V, EL> CacheItem<K, V, EL>
-where
-    K: Key,
-    V: Value,
-    EL: Link,
-{
-    pub fn new(key: K, value: V) -> Self {
-        Self {
-            elink: EL::default(),
-            clink: HashMapLink::default(),
-            key,
-            value,
-
-            priority: 0,
-        }
-    }
-
-    pub fn key(&self) -> &K {
-        &self.key
-    }
-
-    pub fn value(&self) -> &V {
-        &self.value
-    }
-}
-
-intrusive_adapter! { pub CacheItemEpAdapter<K, V, EL> = Arc<CacheItem<K, V, EL>>: CacheItem<K, V, EL> { elink: EL } where K: Key, V: Value, EL: Link }
-key_adapter! { CacheItemEpAdapter<K, V, EL> = CacheItem<K, V, EL> { key: K } where K: Key, V: Value, EL: Link }
-priority_adapter! { CacheItemEpAdapter<K, V, EL> = CacheItem<K, V, EL> { priority: usize } where K: Key, V: Value, EL: Link }
-
-intrusive_adapter! { pub CacheItemHmAdapter<K, V, EL> = Arc<CacheItem<K, V, EL>>: CacheItem<K, V, EL> { clink: HashMapLink } where K: Key, V: Value, EL: Link }
-key_adapter! { CacheItemHmAdapter<K, V, EL> = CacheItem<K, V, EL> { key: K } where K: Key, V: Value, EL: Link }
-
-impl<K, V, E, EL> Cache<K, V, E, EL>
-where
-    K: Key,
-    V: Value,
-    E: EvictionPolicy<Adapter = CacheItemEpAdapter<K, V, EL>>,
-    EL: Link,
-{
-    pub fn new(config: CacheConfig<E>) -> Self {
-        let mut shards = Vec::with_capacity(1 << config.shard_bits);
-
-        let shard_capacity = config.capacity / (1 << config.shard_bits);
-
-        for _ in 0..(1 << config.shard_bits) {
-            let shard = CacheShard {
-                container: HashMap::new(config.hashmap_bits),
-                eviction: E::new(config.eviction_config.clone()),
-                capacity: shard_capacity,
-                size: 0,
-            };
-            shards.push(Mutex::new(shard));
-        }
-
-        Self { shards }
-    }
-
-    pub fn insert(&self, key: K, value: V) -> Vec<Arc<CacheItem<K, V, EL>>> {
-        let hash = self.hash_key(&key);
-        let slot = (self.shards.len() - 1) & hash as usize;
-        let weight = key.weight() + value.weight();
-
-        let item = Arc::new(CacheItem::new(key, value));
-
-        let mut shard = self.shards[slot].lock();
-
-        let to_evict = {
-            let mut to_evict = vec![];
-            let mut to_evict_size = 0;
-            for item in shard.eviction.iter() {
-                if shard.size + weight - to_evict_size <= shard.capacity {
-                    break;
-                }
-                to_evict.push(item.clone());
-                to_evict_size += item.key.weight() + item.value.weight();
-            }
-            to_evict
-        };
-        for item in to_evict.iter() {
-            shard.eviction.remove(item);
-            unsafe { shard.container.remove_in_place(item.clink.raw()) };
-        }
-
-        shard.container.insert(item.clone());
-        shard.eviction.insert(item);
-        shard.size += weight;
-
-        to_evict
-    }
-
-    pub fn remove(&self, key: &K) -> Option<Arc<CacheItem<K, V, EL>>> {
-        let hash = self.hash_key(key);
-        let slot = (self.shards.len() - 1) & hash as usize;
-
-        let mut shard = self.shards[slot].lock();
-
-        let item = shard.container.remove(key);
-        if let Some(item) = &item {
-            shard.eviction.remove(item);
-        }
-        item
-    }
-
-    pub fn lookup(&self, key: &K) -> Option<Arc<CacheItem<K, V, EL>>> {
-        let hash = self.hash_key(key);
-        let slot = (self.shards.len() - 1) & hash as usize;
-
-        let mut shard = self.shards[slot].lock();
-        match shard.container.lookup(key) {
-            Some(item) => {
-                let item = unsafe {
-                    Arc::increment_strong_count(item as *const _);
-                    Arc::from_raw(item as *const _)
-                };
-                shard.eviction.access(&item);
-                Some(item)
-            }
-            None => None,
-        }
-    }
-
-    fn hash_key(&self, key: &K) -> u64 {
-        let mut hasher = XxHash64::default();
-        key.hash(&mut hasher);
-        hasher.finish()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use foyer_intrusive::eviction::sfifo::{SegmentedFifo, SegmentedFifoConfig, SegmentedFifoLink};
-
-    use super::*;
-
-    #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-    struct K(usize);
-
-    impl Key for K {
-        fn weight(&self) -> usize {
-            0
-        }
-    }
-
-    #[derive(Debug, PartialEq, Eq, Clone)]
-    struct V(usize);
-
-    impl Value for V {
-        fn weight(&self) -> usize {
-            self.0
-        }
-    }
-
-    type FifoCacheConfig = CacheConfig<SegmentedFifo<CacheItemEpAdapter<K, V, SegmentedFifoLink>>>;
-    type FifoCache = Cache<K, V, SegmentedFifo<CacheItemEpAdapter<K, V, SegmentedFifoLink>>, SegmentedFifoLink>;
-
-    #[test]
-    fn test_fifo_cache_simple() {
-        let config = FifoCacheConfig {
-            capacity: 10,
-            shard_bits: 0, // 1 shard
-            hashmap_bits: 6,
-            eviction_config: SegmentedFifoConfig {
-                segment_ratios: vec![1],
-            },
-        };
-        let cache = FifoCache::new(config);
-
-        // cache: 1, 2, 3, 4
-        for i in 1..=4 {
-            let evicted = cache.insert(K(i), V(i));
-            assert!(evicted.is_empty());
-        }
-        for i in 1..=4 {
-            let item = cache.lookup(&K(i)).unwrap();
-            assert_eq!(item.key(), &K(i));
-            assert_eq!(item.value(), &V(i));
-        }
-
-        // cache: 4, 5
-        // evicted: 1, 2, 3
-        let evicted = cache.insert(K(5), V(5));
-        assert_eq!(evicted.len(), 3);
-        for (i, item) in evicted.into_iter().enumerate() {
-            assert_eq!(Arc::strong_count(&item), 1);
-            assert_eq!(item.key().0, i + 1);
-            assert_eq!(item.value().0, i + 1);
-        }
-
-        // lookup: 5
-        // cache: 4
-        // removed: 5
-        let res = cache.lookup(&K(5)).unwrap();
-        let item = cache.remove(&K(5)).unwrap();
-        assert_eq!(item.key(), &K(5));
-        assert_eq!(item.value(), &V(5));
-        assert_eq!(Arc::strong_count(&item), 2);
-        drop(res);
-        assert_eq!(Arc::strong_count(&item), 1);
-
-        // cache: 10
-        // evicted: 4
-        let evicted = cache.insert(K(10), V(10));
-        assert_eq!(evicted.len(), 1);
-        assert_eq!(evicted[0].key(), &K(4));
-        assert_eq!(evicted[0].value(), &V(4));
-        assert_eq!(Arc::strong_count(&evicted[0]), 1);
-    }
-}
+#![feature(let_chains)]
+#![feature(lint_reasons)]
+
+//! This crate provides a concurrent in-memory cache component that supports replaceable eviction algorithm.
+//!
+//! # Motivation
+//!
+//! There are a few goals to achieve with the crate:
+//!
+//! 1. Pluggable eviction algorithm with the same abstraction.
+//! 2. Tracking the real memory usage by the cache. Including both holding by the cache and by the external users.
+//! 3. Reduce the concurrent read-through requests into one.
+//!
+//! To achieve them, the crate needs to combine the advantages of the implementations of RocksDB and CacheLib.
+//!
+//! # Design
+//!
+//! The cache is mainly compused of the following components:
+//! 1. handle             : Carries the cached entry, reference count, pointer links in the eviction container, etc.
+//! 2. indexer            : Indexes cached keys to the handles.
+//! 3. eviction container : Defines the order of eviction. Usually implemented with intrusive data structures.
+//!
+//! Because a handle needs to be referenced and mutated by both the indexer and the eviction container in the same
+//! thread, it is hard to implement in 100% safe Rust without overhead. So, the APIs of the indexer and the eviciton
+//! container are defined with `NonNull` pointers of the handles.
+//!
+//! When some entry is inserted into the cache, the associated handle should be transmuted into pointer without
+//! dropping. When some entry is removed from the cache, the pointer of the associated handle should be transmuted into
+//! an owned data structure.
+//!
+//! # Handle Lifetime
+//!
+//! The handle is created during a new entry is being inserted, and then inserted into both the indexer and the eviction
+//! container.
+//!
+//! The handle is return if the entry is retrieved from the cache. The handle will track the count of the external
+//! owners to decide the time to reclaim.
+//!
+//! When a key is removed or updated, the original handle will be removed from the indexer and the eviction container,
+//! and waits to be released by all the external owners before reclamation.
+//!
+//! When the cache is full and being inserted, a handle will be evicted from the eviction container based on the
+//! eviction algorithm. The evicted handle will NOT be removed from the indexer immediately because it still occupies
+//! memory and can be used by queries followed up.
+//!
+//! After the handle is released by all the external owners, the eviction container will update its order or evict it
+//! based on the eviction algorithm. If it doesn't appear in the eviction container, it may be reinserted if it still in
+//! the indexer and there is enough space. Otherwise, it will be removed from both the indexer and the eviction
+//! container.
+//!
+//! The handle that does not appear in either the indexer or the eviction container, and has no external owner, will be
+//! destroyed.
+
+pub mod cache;
+pub mod eviction;
+pub mod handle;
+pub mod indexer;
+
+use std::hash::Hash;
+
+pub trait Key: Send + Sync + 'static + Hash + Eq + Ord {}
+pub trait Value: Send + Sync + 'static {}
+
+impl<T: Send + Sync + 'static + Hash + Eq + Ord> Key for T {}
+impl<T: Send + Sync + 'static> Value for T {}
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index 4d7cedb0..bde487bf 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use foyer_common::queue::AsyncQueue;
+use foyer_common::async_queue::AsyncQueue;
 use foyer_intrusive::{
     core::adapter::Link,
     eviction::{EvictionPolicy, EvictionPolicyExt},
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 2b30e528..76dac42e 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -17,6 +17,7 @@ publish = true
 
 ### BEGIN HAKARI SECTION
 [dependencies]
+ahash = { version = "0.8" }
 crossbeam-channel = { version = "0.5" }
 crossbeam-utils = { version = "0.8" }
 either = { version = "1", default-features = false, features = ["use_std"] }
@@ -25,8 +26,8 @@ futures-core = { version = "0.3" }
 futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
+hashbrown = { version = "0.14", features = ["raw"] }
 libc = { version = "0.2", features = ["extra_traits"] }
-memchr = { version = "2" }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
 rand = { version = "0.8", features = ["small_rng"] }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index dadb9798..e3dc1741 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -15,5 +15,6 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 foyer-common = { version = "0.3", path = "../foyer-common" }
 foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
+foyer-memory = { version = "0.1", path = "../foyer-memory" }
 foyer-storage = { version = "0.4", path = "../foyer-storage" }
 foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index e15cb512..6a90482b 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -17,4 +17,5 @@
 
 pub use foyer_common as common;
 pub use foyer_intrusive as intrusive;
+pub use foyer_memory as memory;
 pub use foyer_storage as storage;

From 0d5ee4c18c8e1b378b97274dac6cc0cb5a1e6d8f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 7 Mar 2024 11:50:54 +0800
Subject: [PATCH 220/261] feat: introduce metrics for in-memory cache (#283)

* feat: introduce foyer-memory Cache abstraction

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove old foyer-memory code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce base handle

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: make cache abstraction use handle trait with base handle

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: impl fifo eviction

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add basic tests

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refactor directory hierarchy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: call access when get

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make hakari happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix dangling ptr after remove

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce RemovableQueue to make FIFO impl easilier.

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use removable queue to simplify fifo impl

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix removable queue grow token updates

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: tiny refactors

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use full type name for trait associated type

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce associate type Context for handle

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce lru eviction policy for foyer-memory

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: add a simple ut and fix bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused generic type

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use rocksdb lru impl instead of cachelib lru

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: add waiters and entry API for request dedup

TODO: add ut for Cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce new in-memory cache abstraction into foyer-memory

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use intrusive dlist to impl fifo

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use intrusive dlist to impl LRU with high-pri pool

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix reinsert bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add ut and fix some bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: expose foyer-memory via foyer crate

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: allow entry future returns context besides

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make lru context clone and copy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make lru context derive Eq and PartialEq

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: impl Future for Entry

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused comments

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* doc: add crate document

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add more uts for cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add ut to cover not reinsert for full

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix typo

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: add metrics shard for in-memory cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use atomic counter for metrics

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs   | 68 +++++++++++++++++++++++++++++++------
 foyer-memory/src/lib.rs     |  1 +
 foyer-memory/src/metrics.rs | 44 ++++++++++++++++++++++++
 3 files changed, 102 insertions(+), 11 deletions(-)
 create mode 100644 foyer-memory/src/metrics.rs

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index c5c7370b..1c76dfa2 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -39,9 +39,16 @@ use crate::{
     },
     handle::Handle,
     indexer::{HashTableIndexer, Indexer},
+    metrics::Metrics,
     Key, Value,
 };
 
+struct CacheContext<T> {
+    metrics: Metrics,
+    /// The object pool to avoid frequent handle allocating, shared by all shards.
+    object_pool: ArrayQueue<Box<T>>,
+}
+
 #[expect(clippy::type_complexity)]
 struct CacheShard<K, V, H, E, I, S>
 where
@@ -60,8 +67,7 @@ where
 
     waiters: HashMap<K, Vec<oneshot::Sender<CacheEntry<K, V, H, E, I, S>>>>,
 
-    /// The object pool to avoid frequent handle allocating, shared by all shards.
-    object_pool: Arc<ArrayQueue<Box<H>>>,
+    context: Arc<CacheContext<H>>,
 }
 
 impl<K, V, H, E, I, S> CacheShard<K, V, H, E, I, S>
@@ -73,7 +79,7 @@ where
     I: Indexer<Key = K, Handle = H>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    fn new(config: &CacheConfig<E, S>, usage: Arc<AtomicUsize>, object_pool: Arc<ArrayQueue<Box<H>>>) -> Self {
+    fn new(config: &CacheConfig<E, S>, usage: Arc<AtomicUsize>, context: Arc<CacheContext<H>>) -> Self {
         let indexer = I::new();
         let eviction = unsafe { E::new(config) };
         let capacity = config.capacity / config.shards;
@@ -84,7 +90,7 @@ where
             capacity,
             usage,
             waiters,
-            object_pool,
+            context,
         }
     }
 
@@ -98,7 +104,7 @@ where
         context: H::Context,
         last_reference_entries: &mut Vec<(K, V)>,
     ) -> NonNull<H> {
-        let mut handle = self.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
+        let mut handle = self.context.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
         handle.init(hash, key, value, charge, context);
         let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
 
@@ -106,6 +112,8 @@ where
 
         debug_assert!(!ptr.as_ref().base().is_in_indexer());
         if let Some(old) = self.indexer.insert(ptr) {
+            self.context.metrics.replace.fetch_add(1, Ordering::Relaxed);
+
             debug_assert!(!old.as_ref().base().is_in_indexer());
             if old.as_ref().base().is_in_eviction() {
                 self.eviction.remove(old);
@@ -115,6 +123,8 @@ where
             if let Some(entry) = self.try_release_handle(old, false) {
                 last_reference_entries.push(entry);
             }
+        } else {
+            self.context.metrics.insert.fetch_add(1, Ordering::Relaxed);
         }
         self.eviction.push(ptr);
 
@@ -128,7 +138,16 @@ where
     }
 
     unsafe fn get(&mut self, hash: u64, key: &K) -> Option<NonNull<H>> {
-        let mut ptr = self.indexer.get(hash, key)?;
+        let mut ptr = match self.indexer.get(hash, key) {
+            Some(ptr) => {
+                self.context.metrics.hit.fetch_add(1, Ordering::Relaxed);
+                ptr
+            }
+            None => {
+                self.context.metrics.miss.fetch_add(1, Ordering::Relaxed);
+                return None;
+            }
+        };
         let base = ptr.as_mut().base_mut();
         debug_assert!(base.is_in_indexer());
 
@@ -143,6 +162,7 @@ where
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
     unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V)> {
         let ptr = self.indexer.remove(hash, key)?;
+        self.context.metrics.remove.fetch_add(1, Ordering::Relaxed);
         if ptr.as_ref().base().is_in_eviction() {
             self.eviction.remove(ptr);
         }
@@ -165,6 +185,8 @@ where
             debug_assert!((&eptrs - &ptrs).is_empty());
         }
 
+        self.context.metrics.remove.fetch_add(ptrs.len(), Ordering::Relaxed);
+
         // The handles in the indexer covers the handles in the eviction container.
         // So only the handles drained from the indexer need to be released.
         for ptr in ptrs {
@@ -179,6 +201,7 @@ where
         while self.usage.load(Ordering::Relaxed) + charge > self.capacity
             && let Some(evicted) = self.eviction.pop()
         {
+            self.context.metrics.evict.fetch_add(1, Ordering::Relaxed);
             let base = evicted.as_ref().base();
             debug_assert!(base.is_in_indexer());
             debug_assert!(!base.is_in_eviction());
@@ -218,8 +241,12 @@ where
             // the cache shard cannot release enough charges for the new inserted entries.
             // In this case, the reinsertion should be given up.
             if reinsert && self.usage.load(Ordering::Relaxed) <= self.capacity {
+                let was_in_eviction = base.is_in_eviction();
                 self.eviction.reinsert(ptr);
                 if ptr.as_ref().base().is_in_eviction() {
+                    if was_in_eviction {
+                        self.context.metrics.reinsert.fetch_add(1, Ordering::Relaxed);
+                    }
                     return None;
                 }
             }
@@ -236,11 +263,13 @@ where
         debug_assert!(!base.is_in_eviction());
         debug_assert!(!base.has_refs());
 
+        self.context.metrics.release.fetch_add(1, Ordering::Relaxed);
+
         self.usage.fetch_sub(base.charge(), Ordering::Relaxed);
         let entry = base.take();
 
         let handle = Box::from_raw(ptr.as_ptr());
-        let _ = self.object_pool.push(handle);
+        let _ = self.context.object_pool.push(handle);
 
         Some(entry)
     }
@@ -344,6 +373,8 @@ where
     capacity: usize,
     usages: Vec<Arc<AtomicUsize>>,
 
+    context: Arc<CacheContext<H>>,
+
     hash_builder: S,
 }
 
@@ -358,10 +389,14 @@ where
 {
     pub fn new(config: CacheConfig<E, S>) -> Self {
         let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
-        let object_pool = Arc::new(ArrayQueue::new(config.object_pool_capacity));
+        let context = Arc::new(CacheContext {
+            metrics: Metrics::default(),
+            object_pool: ArrayQueue::new(config.object_pool_capacity),
+        });
+
         let shards = usages
             .iter()
-            .map(|usage| CacheShard::new(&config, usage.clone(), object_pool.clone()))
+            .map(|usage| CacheShard::new(&config, usage.clone(), context.clone()))
             .map(Mutex::new)
             .collect_vec();
 
@@ -369,6 +404,7 @@ where
             shards,
             capacity: config.capacity,
             usages,
+            context,
             hash_builder: config.hash_builder,
         }
     }
@@ -459,6 +495,10 @@ where
         self.usages.iter().map(|usage| usage.load(Ordering::Relaxed)).sum()
     }
 
+    pub fn metrics(&self) -> &Metrics {
+        &self.context.metrics
+    }
+
     unsafe fn try_release_external_handle(&self, ptr: NonNull<H>) {
         let entry = {
             let base = ptr.as_ref().base();
@@ -498,7 +538,7 @@ where
                     ptr,
                 });
             }
-            match shard.waiters.entry(key.clone()) {
+            let entry = match shard.waiters.entry(key.clone()) {
                 HashMapEntry::Occupied(mut o) => {
                     let (tx, rx) = oneshot::channel();
                     o.get_mut().push(tx);
@@ -528,7 +568,13 @@ where
                     });
                     Entry::Miss(join)
                 }
-            }
+            };
+            match entry {
+                Entry::Wait(_) => shard.context.metrics.queue.fetch_add(1, Ordering::Relaxed),
+                Entry::Miss(_) => shard.context.metrics.fetch.fetch_add(1, Ordering::Relaxed),
+                _ => unreachable!(),
+            };
+            entry
         }
     }
 }
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 8ff8a433..280551ec 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -69,6 +69,7 @@ pub mod cache;
 pub mod eviction;
 pub mod handle;
 pub mod indexer;
+pub mod metrics;
 
 use std::hash::Hash;
 
diff --git a/foyer-memory/src/metrics.rs b/foyer-memory/src/metrics.rs
new file mode 100644
index 00000000..a5bcbf3c
--- /dev/null
+++ b/foyer-memory/src/metrics.rs
@@ -0,0 +1,44 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::atomic::AtomicUsize;
+
+#[derive(Debug, Default)]
+pub struct Metrics {
+    /// successful inserts without replaces
+    pub insert: AtomicUsize,
+    /// successful replaces
+    pub replace: AtomicUsize,
+
+    /// get hits
+    pub hit: AtomicUsize,
+    /// get misses
+    pub miss: AtomicUsize,
+
+    /// fetches after cache miss with `entry` interface
+    pub fetch: AtomicUsize,
+    /// deduped fetches after cache miss with `entry` interface
+    pub queue: AtomicUsize,
+
+    /// successful removes
+    pub remove: AtomicUsize,
+
+    /// evicts from the eviction container
+    pub evict: AtomicUsize,
+    /// successful reinserts, only counts successful reinserts after evicted
+    pub reinsert: AtomicUsize,
+
+    /// released handles
+    pub release: AtomicUsize,
+}

From e8f884295e96645731b07df235c72acddb665921 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 7 Mar 2024 15:51:01 +0800
Subject: [PATCH 221/261] feat: introduce event listener for foyer-memory cache
 (#279)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs         | 162 ++++++++++++++++++------------
 foyer-memory/src/event.rs         |  58 +++++++++++
 foyer-memory/src/eviction/fifo.rs |  17 +---
 foyer-memory/src/eviction/lru.rs  |  28 ++----
 foyer-memory/src/eviction/mod.rs  |   6 +-
 foyer-memory/src/handle.rs        |   9 +-
 foyer-memory/src/lib.rs           |   1 +
 7 files changed, 182 insertions(+), 99 deletions(-)
 create mode 100644 foyer-memory/src/event.rs

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 1c76dfa2..e7755534 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -32,9 +32,10 @@ use parking_lot::Mutex;
 use tokio::{sync::oneshot, task::JoinHandle};
 
 use crate::{
+    event::{CacheEventListener, DefaultCacheEventListener},
     eviction::{
-        fifo::{Fifo, FifoHandle},
-        lru::{Lru, LruHandle},
+        fifo::{Fifo, FifoContext, FifoHandle},
+        lru::{Lru, LruContext, LruHandle},
         Eviction,
     },
     handle::Handle,
@@ -43,20 +44,22 @@ use crate::{
     Key, Value,
 };
 
-struct CacheContext<T> {
+struct CacheContext<T, L> {
     metrics: Metrics,
     /// The object pool to avoid frequent handle allocating, shared by all shards.
     object_pool: ArrayQueue<Box<T>>,
+    listener: L,
 }
 
 #[expect(clippy::type_complexity)]
-struct CacheShard<K, V, H, E, I, S>
+struct CacheShard<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     indexer: I,
@@ -65,24 +68,29 @@ where
     capacity: usize,
     usage: Arc<AtomicUsize>,
 
-    waiters: HashMap<K, Vec<oneshot::Sender<CacheEntry<K, V, H, E, I, S>>>>,
+    waiters: HashMap<K, Vec<oneshot::Sender<CacheEntry<K, V, H, E, I, L, S>>>>,
 
-    context: Arc<CacheContext<H>>,
+    context: Arc<CacheContext<H, L>>,
 }
 
-impl<K, V, H, E, I, S> CacheShard<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> CacheShard<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    fn new(config: &CacheConfig<E, S>, usage: Arc<AtomicUsize>, context: Arc<CacheContext<H>>) -> Self {
+    fn new(
+        capacity: usize,
+        eviction_config: &E::Config,
+        usage: Arc<AtomicUsize>,
+        context: Arc<CacheContext<H, L>>,
+    ) -> Self {
         let indexer = I::new();
-        let eviction = unsafe { E::new(config) };
-        let capacity = config.capacity / config.shards;
+        let eviction = unsafe { E::new(capacity, eviction_config) };
         let waiters = HashMap::default();
         Self {
             indexer,
@@ -102,7 +110,7 @@ where
         value: V,
         charge: usize,
         context: H::Context,
-        last_reference_entries: &mut Vec<(K, V)>,
+        last_reference_entries: &mut Vec<(K, V, H::Context, usize)>,
     ) -> NonNull<H> {
         let mut handle = self.context.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
         handle.init(hash, key, value, charge, context);
@@ -160,7 +168,7 @@ where
     /// Remove a key from the cache.
     ///
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V)> {
+    unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V, H::Context, usize)> {
         let ptr = self.indexer.remove(hash, key)?;
         self.context.metrics.remove.fetch_add(1, Ordering::Relaxed);
         if ptr.as_ref().base().is_in_eviction() {
@@ -172,7 +180,7 @@ where
     }
 
     /// Clear all cache entries.
-    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V)>) {
+    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
         // TODO(MrCroxx): Avoid collecting here?
         let ptrs = self.indexer.drain().collect_vec();
         let eptrs = self.eviction.clear();
@@ -197,7 +205,7 @@ where
         }
     }
 
-    unsafe fn evict(&mut self, charge: usize, last_reference_entries: &mut Vec<(K, V)>) {
+    unsafe fn evict(&mut self, charge: usize, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
         while self.usage.load(Ordering::Relaxed) + charge > self.capacity
             && let Some(evicted) = self.eviction.pop()
         {
@@ -214,7 +222,7 @@ where
     /// Release a handle used by an external user.
     ///
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn try_release_external_handle(&mut self, mut ptr: NonNull<H>) -> Option<(K, V)> {
+    unsafe fn try_release_external_handle(&mut self, mut ptr: NonNull<H>) -> Option<(K, V, H::Context, usize)> {
         ptr.as_mut().base_mut().dec_refs();
         self.try_release_handle(ptr, true)
     }
@@ -224,7 +232,7 @@ where
     /// Return the entry if the handle is released.
     ///
     /// Recycle it if possible.
-    unsafe fn try_release_handle(&mut self, mut ptr: NonNull<H>, reinsert: bool) -> Option<(K, V)> {
+    unsafe fn try_release_handle(&mut self, mut ptr: NonNull<H>, reinsert: bool) -> Option<(K, V, H::Context, usize)> {
         let base = ptr.as_mut().base_mut();
 
         if base.has_refs() {
@@ -275,13 +283,14 @@ where
     }
 }
 
-impl<K, V, H, E, I, S> Drop for CacheShard<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> Drop for CacheShard<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn drop(&mut self) {
@@ -289,9 +298,10 @@ where
     }
 }
 
-pub struct CacheConfig<E, S = RandomState>
+pub struct CacheConfig<E, L, S = RandomState>
 where
     E: Eviction,
+    L: CacheEventListener,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub capacity: usize,
@@ -299,32 +309,35 @@ where
     pub eviction_config: E::Config,
     pub object_pool_capacity: usize,
     pub hash_builder: S,
+    pub event_listener: L,
 }
 
 #[expect(clippy::type_complexity)]
-pub enum Entry<K, V, H, E, I, S, ER>
+pub enum Entry<K, V, H, E, I, L, S, ER>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
 {
     Invalid,
-    Hit(CacheEntry<K, V, H, E, I, S>),
-    Wait(oneshot::Receiver<CacheEntry<K, V, H, E, I, S>>),
-    Miss(JoinHandle<std::result::Result<CacheEntry<K, V, H, E, I, S>, ER>>),
+    Hit(CacheEntry<K, V, H, E, I, L, S>),
+    Wait(oneshot::Receiver<CacheEntry<K, V, H, E, I, L, S>>),
+    Miss(JoinHandle<std::result::Result<CacheEntry<K, V, H, E, I, L, S>, ER>>),
 }
 
-impl<K, V, H, E, I, S, ER> Default for Entry<K, V, H, E, I, S, ER>
+impl<K, V, H, E, I, L, S, ER> Default for Entry<K, V, H, E, I, L, S, ER>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
 {
@@ -333,17 +346,18 @@ where
     }
 }
 
-impl<K, V, H, E, I, S, ER> Future for Entry<K, V, H, E, I, S, ER>
+impl<K, V, H, E, I, L, S, ER> Future for Entry<K, V, H, E, I, L, S, ER>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error + From<oneshot::error::RecvError>,
 {
-    type Output = std::result::Result<CacheEntry<K, V, H, E, I, S>, ER>;
+    type Output = std::result::Result<CacheEntry<K, V, H, E, I, L, S>, ER>;
 
     fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
         match &mut *self {
@@ -359,44 +373,49 @@ where
 }
 
 #[expect(clippy::type_complexity)]
-pub struct Cache<K, V, H, E, I, S = RandomState>
+pub struct Cache<K, V, H, E, I, L, S = RandomState>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    shards: Vec<Mutex<CacheShard<K, V, H, E, I, S>>>,
+    shards: Vec<Mutex<CacheShard<K, V, H, E, I, L, S>>>,
 
     capacity: usize,
     usages: Vec<Arc<AtomicUsize>>,
 
-    context: Arc<CacheContext<H>>,
+    context: Arc<CacheContext<H, L>>,
 
     hash_builder: S,
 }
 
-impl<K, V, H, E, I, S> Cache<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> Cache<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn new(config: CacheConfig<E, S>) -> Self {
+    pub fn new(config: CacheConfig<E, L, S>) -> Self {
         let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
         let context = Arc::new(CacheContext {
             metrics: Metrics::default(),
             object_pool: ArrayQueue::new(config.object_pool_capacity),
+            listener: config.event_listener,
         });
 
+        let shard_capacity = config.capacity / config.shards;
+
         let shards = usages
             .iter()
-            .map(|usage| CacheShard::new(&config, usage.clone(), context.clone()))
+            .map(|usage| CacheShard::new(shard_capacity, &config.eviction_config, usage.clone(), context.clone()))
             .map(Mutex::new)
             .collect_vec();
 
@@ -409,7 +428,7 @@ where
         }
     }
 
-    pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> CacheEntry<K, V, H, E, I, S> {
+    pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> CacheEntry<K, V, H, E, I, L, S> {
         self.insert_with_context(key, value, charge, H::Context::default())
     }
 
@@ -419,7 +438,7 @@ where
         value: V,
         charge: usize,
         context: H::Context,
-    ) -> CacheEntry<K, V, H, E, I, S> {
+    ) -> CacheEntry<K, V, H, E, I, L, S> {
         let hash = self.hash_builder.hash_one(&key);
 
         let mut to_deallocate = vec![];
@@ -448,8 +467,9 @@ where
         }
 
         // Do not deallocate data within the lock section.
-        // TODO: call listener here.
-        drop(to_deallocate);
+        for (key, value, context, charges) in to_deallocate {
+            self.context.listener.on_release(key, value, context, charges)
+        }
 
         entry
     }
@@ -457,17 +477,18 @@ where
     pub fn remove(&self, key: &K) {
         let hash = self.hash_builder.hash_one(key);
 
-        let kv = unsafe {
+        let entry = unsafe {
             let mut shard = self.shards[hash as usize % self.shards.len()].lock();
             shard.remove(hash, key)
         };
 
         // Do not deallocate data within the lock section.
-        // TODO: call listener here.
-        drop(kv);
+        if let Some((key, value, context, charges)) = entry {
+            self.context.listener.on_release(key, value, context, charges);
+        }
     }
 
-    pub fn get(self: &Arc<Self>, key: &K) -> Option<CacheEntry<K, V, H, E, I, S>> {
+    pub fn get(self: &Arc<Self>, key: &K) -> Option<CacheEntry<K, V, H, E, I, L, S>> {
         let hash = self.hash_builder.hash_one(key);
 
         unsafe {
@@ -507,22 +528,24 @@ where
         };
 
         // Do not deallocate data within the lock section.
-        // TODO: call listener here.
-        drop(entry);
+        if let Some((key, value, context, charges)) = entry {
+            self.context.listener.on_release(key, value, context, charges);
+        }
     }
 }
 
 // TODO(MrCroxx): use `hashbrown::HashTable` with `Handle` may relax the `Clone` bound?
-impl<K, V, H, E, I, S> Cache<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> Cache<K, V, H, E, I, L, S>
 where
     K: Key + Clone,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> Entry<K, V, H, E, I, S, ER>
+    pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> Entry<K, V, H, E, I, L, S, ER>
     where
         F: FnOnce() -> FU,
         FU: Future<Output = std::result::Result<(V, usize, Option<H::Context>), ER>> + Send + 'static,
@@ -579,26 +602,28 @@ where
     }
 }
 
-pub struct CacheEntry<K, V, H, E, I, S = RandomState>
+pub struct CacheEntry<K, V, H, E, I, L, S = RandomState>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    cache: Arc<Cache<K, V, H, E, I, S>>,
+    cache: Arc<Cache<K, V, H, E, I, L, S>>,
     ptr: NonNull<H>,
 }
 
-impl<K, V, H, E, I, S> CacheEntry<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> CacheEntry<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub fn key(&self) -> &H::Key {
@@ -622,13 +647,14 @@ where
     }
 }
 
-impl<K, V, H, E, I, S> Clone for CacheEntry<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> Clone for CacheEntry<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn clone(&self) -> Self {
@@ -647,13 +673,14 @@ where
     }
 }
 
-impl<K, V, H, E, I, S> Drop for CacheEntry<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> Drop for CacheEntry<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn drop(&mut self) {
@@ -661,13 +688,14 @@ where
     }
 }
 
-impl<K, V, H, E, I, S> Deref for CacheEntry<K, V, H, E, I, S>
+impl<K, V, H, E, I, L, S> Deref for CacheEntry<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     type Target = V;
@@ -677,45 +705,52 @@ where
     }
 }
 
-unsafe impl<K, V, H, E, I, S> Send for CacheEntry<K, V, H, E, I, S>
+unsafe impl<K, V, H, E, I, L, S> Send for CacheEntry<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
 }
-unsafe impl<K, V, H, E, I, S> Sync for CacheEntry<K, V, H, E, I, S>
+unsafe impl<K, V, H, E, I, L, S> Sync for CacheEntry<K, V, H, E, I, L, S>
 where
     K: Key,
     V: Value,
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
 }
 
-pub type FifoCache<K, V, S = RandomState> =
-    Cache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, S>;
-pub type FifoCacheConfig<K, V, S = RandomState> = CacheConfig<Fifo<K, V>, S>;
-pub type FifoCacheEntry<K, V, S = RandomState> =
-    CacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, S>;
+pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+    Cache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+    CacheConfig<Fifo<K, V>, L, S>;
+pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+    CacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
 
-pub type LruCache<K, V, S = RandomState> =
-    Cache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, S>;
-pub type LruCacheConfig<K, V, S = RandomState> = CacheConfig<Lru<K, V>, S>;
-pub type LruCacheEntry<K, V, S = RandomState> =
-    CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, S>;
+pub type LruCache<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+    Cache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+    CacheConfig<Lru<K, V>, L, S>;
+pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+    CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
 
 #[cfg(test)]
 mod tests {
     use rand::{rngs::SmallRng, RngCore, SeedableRng};
 
     use super::*;
-    use crate::eviction::{fifo::FifoConfig, lru::LruConfig, test_utils::TestEviction};
+    use crate::{
+        event::DefaultCacheEventListener,
+        eviction::{fifo::FifoConfig, lru::LruConfig, test_utils::TestEviction},
+    };
 
     fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
@@ -737,6 +772,7 @@ mod tests {
             eviction_config: FifoConfig {},
             object_pool_capacity: 16,
             hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
         };
         let cache = Arc::new(FifoCache::<u64, u64>::new(config));
 
@@ -760,6 +796,7 @@ mod tests {
             eviction_config: FifoConfig {},
             object_pool_capacity: 1,
             hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
         };
         Arc::new(FifoCache::<u64, String>::new(config))
     }
@@ -773,6 +810,7 @@ mod tests {
             },
             object_pool_capacity: 1,
             hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
         };
         Arc::new(LruCache::<u64, String>::new(config))
     }
diff --git a/foyer-memory/src/event.rs b/foyer-memory/src/event.rs
new file mode 100644
index 00000000..c5b9c348
--- /dev/null
+++ b/foyer-memory/src/event.rs
@@ -0,0 +1,58 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::marker::PhantomData;
+
+use crate::{Key, Value};
+
+pub trait CacheEventListener: Send + Sync + 'static {
+    type Key: Key;
+    type Value: Value;
+    type Context: Send + Sync + 'static;
+
+    /// The function is called when an entry is released by the cache and all external users.
+    ///
+    /// The arguments includes the key, value and context with ownership.
+    fn on_release(&self, key: Self::Key, value: Self::Value, context: Self::Context, charges: usize);
+}
+
+pub struct DefaultCacheEventListener<K, V, C>(PhantomData<(K, V, C)>)
+where
+    K: Key,
+    V: Value,
+    C: Send + Sync + 'static;
+
+impl<K, V, C> Default for DefaultCacheEventListener<K, V, C>
+where
+    K: Key,
+    V: Value,
+    C: Send + Sync + 'static,
+{
+    fn default() -> Self {
+        Self(Default::default())
+    }
+}
+
+impl<K, V, C> CacheEventListener for DefaultCacheEventListener<K, V, C>
+where
+    K: Key,
+    V: Value,
+    C: Send + Sync + 'static,
+{
+    type Key = K;
+    type Value = V;
+    type Context = C;
+
+    fn on_release(&self, _key: Self::Key, _value: Self::Value, _context: Self::Context, _charges: usize) {}
+}
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index 761ad112..f0180ea5 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, hash::BuildHasher, ptr::NonNull};
+use std::{fmt::Debug, ptr::NonNull};
 
 use foyer_intrusive::{
     collections::dlist::{Dlist, DlistLink},
@@ -20,7 +20,6 @@ use foyer_intrusive::{
 };
 
 use crate::{
-    cache::CacheConfig,
     eviction::Eviction,
     handle::{BaseHandle, Handle},
     Key, Value,
@@ -97,7 +96,7 @@ where
     type Handle = FifoHandle<K, V>;
     type Config = FifoConfig;
 
-    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(_: &CacheConfig<Self, S>) -> Self
+    unsafe fn new(_capacity: usize, _config: &Self::Config) -> Self
     where
         Self: Sized,
     {
@@ -159,7 +158,7 @@ where
 
 #[cfg(test)]
 pub mod tests {
-    use ahash::RandomState;
+
     use itertools::Itertools;
 
     use super::*;
@@ -196,15 +195,7 @@ pub mod tests {
         unsafe {
             let ptrs = (0..8).map(|i| new_test_fifo_handle_ptr(i, i)).collect_vec();
 
-            let config = CacheConfig {
-                capacity: 0,
-                shards: 1,
-                eviction_config: FifoConfig {},
-                object_pool_capacity: 0,
-                hash_builder: RandomState::default(),
-            };
-
-            let mut fifo = TestFifo::new(&config);
+            let mut fifo = TestFifo::new(100, &FifoConfig {});
 
             // 0, 1, 2, 3
             fifo.push(ptrs[0]);
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index 921e2abe..e036bd19 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, hash::BuildHasher, ptr::NonNull};
+use std::{fmt::Debug, ptr::NonNull};
 
 use foyer_intrusive::{
     collections::dlist::{Dlist, DlistLink},
@@ -21,7 +21,6 @@ use foyer_intrusive::{
 };
 
 use crate::{
-    cache::CacheConfig,
     eviction::Eviction,
     handle::{BaseHandle, Handle},
     Key, Value,
@@ -155,20 +154,17 @@ where
     type Handle = LruHandle<K, V>;
     type Config = LruConfig;
 
-    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(config: &CacheConfig<Self, S>) -> Self
+    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
     where
         Self: Sized,
     {
         assert!(
-            config.eviction_config.high_priority_pool_ratio >= 0.0
-                && config.eviction_config.high_priority_pool_ratio <= 1.0,
+            config.high_priority_pool_ratio >= 0.0 && config.high_priority_pool_ratio <= 1.0,
             "high_priority_pool_ratio_percentage must be in [0, 100], given: {}",
-            config.eviction_config.high_priority_pool_ratio
+            config.high_priority_pool_ratio
         );
 
-        let high_priority_charges_capacity =
-            config.capacity as f64 * config.eviction_config.high_priority_pool_ratio / config.shards as f64;
-        let high_priority_charges_capacity = high_priority_charges_capacity as usize;
+        let high_priority_charges_capacity = (capacity as f64 * config.high_priority_pool_ratio) as usize;
 
         Self {
             high_priority_list: Dlist::new(),
@@ -289,7 +285,7 @@ where
 
 #[cfg(test)]
 pub mod tests {
-    use ahash::RandomState;
+
     use foyer_intrusive::core::pointer::Pointer;
     use itertools::Itertools;
 
@@ -353,16 +349,10 @@ pub mod tests {
                 })
                 .collect_vec();
 
-            let config = CacheConfig {
-                capacity: 8,
-                shards: 1,
-                eviction_config: LruConfig {
-                    high_priority_pool_ratio: 0.5,
-                },
-                object_pool_capacity: 0,
-                hash_builder: RandomState::default(),
+            let config = LruConfig {
+                high_priority_pool_ratio: 0.5,
             };
-            let mut lru = TestLru::new(&config);
+            let mut lru = TestLru::new(8, &config);
 
             assert_eq!(lru.high_priority_charges_capacity, 4);
 
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index 98846d2f..3a78e74e 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -12,9 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{hash::BuildHasher, ptr::NonNull};
+use std::ptr::NonNull;
 
-use crate::{cache::CacheConfig, handle::Handle};
+use crate::handle::Handle;
 
 /// The lifetime of `handle: Self::H` is managed by [`Indexer`].
 ///
@@ -26,7 +26,7 @@ pub trait Eviction: Send + Sync + 'static {
     /// Create a new empty eviction container.
     ///
     /// # Safety
-    unsafe fn new<S: BuildHasher + Send + Sync + 'static>(config: &CacheConfig<Self, S>) -> Self
+    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
     where
         Self: Sized;
 
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
index e1be5cdb..f1fea9d3 100644
--- a/foyer-memory/src/handle.rs
+++ b/foyer-memory/src/handle.rs
@@ -94,9 +94,14 @@ where
 
     /// Take key and value from the handle and reset it to the uninited state.
     #[inline(always)]
-    pub fn take(&mut self) -> (K, V) {
+    pub fn take(&mut self) -> (K, V, C, usize) {
         debug_assert!(self.entry.is_some());
-        unsafe { self.entry.take().map(|(key, value, _)| (key, value)).unwrap_unchecked() }
+        unsafe {
+            self.entry
+                .take()
+                .map(|(key, value, context)| (key, value, context, self.charge))
+                .unwrap_unchecked()
+        }
     }
 
     /// Return `true` if the handle is inited.
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 280551ec..0add22eb 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -66,6 +66,7 @@
 //! destroyed.
 
 pub mod cache;
+pub mod event;
 pub mod eviction;
 pub mod handle;
 pub mod indexer;

From ee8a9a9ec0bb3c8cb2cddf673739157f098b4964 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Mar 2024 11:30:59 +0800
Subject: [PATCH 222/261] feat: impl caffeine-like w-tinylfu for in-memory
 cache (#278)

* feat: impl caffeine-like w-tinylfu for in-memory cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix typo

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add unit tests for wtinylfu

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix build after merge

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: expose Lfu in a convenient manner

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix comments

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: upgrade cmsketch, use eps and confidence as config, fix bug

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/eviction/lfu.rs |   2 +-
 foyer-memory/Cargo.toml             |   1 +
 foyer-memory/src/cache.rs           |   8 +
 foyer-memory/src/eviction/lfu.rs    | 564 ++++++++++++++++++++++++++++
 foyer-memory/src/eviction/mod.rs    |   1 +
 5 files changed, 575 insertions(+), 1 deletion(-)
 create mode 100644 foyer-memory/src/eviction/lfu.rs

diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 508fdef2..215e7cfd 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -50,7 +50,7 @@ const ERROR_THRESHOLD: f64 = 5.0;
 const HASH_COUNT: usize = 4;
 const DECAY_FACTOR: f64 = 0.5;
 
-#[derive(Clone, Debug)]
+#[derive(Debug, Clone)]
 pub struct LfuConfig {
     /// The multiplier for window len given the cache size.
     pub window_to_cache_size_ratio: usize,
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 4179e2e4..e3feee65 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -15,6 +15,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 ahash = "0.8"
 bitflags = "2"
+cmsketch = "0.2"
 crossbeam = "0.8"
 foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index e7755534..2721ce39 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -35,6 +35,7 @@ use crate::{
     event::{CacheEventListener, DefaultCacheEventListener},
     eviction::{
         fifo::{Fifo, FifoContext, FifoHandle},
+        lfu::{Lfu, LfuContext, LfuHandle},
         lru::{Lru, LruContext, LruHandle},
         Eviction,
     },
@@ -742,6 +743,13 @@ pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S
 pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
     CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
 
+pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+    Cache<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+    CacheConfig<Lfu<K, V>, L, S>;
+pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+    CacheEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+
 #[cfg(test)]
 mod tests {
     use rand::{rngs::SmallRng, RngCore, SeedableRng};
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
new file mode 100644
index 00000000..89d529f1
--- /dev/null
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -0,0 +1,564 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, ptr::NonNull};
+
+use cmsketch::CMSketchU16;
+use foyer_intrusive::{
+    collections::dlist::{Dlist, DlistLink},
+    core::adapter::Link,
+    intrusive_adapter,
+};
+
+use crate::{
+    eviction::Eviction,
+    handle::{BaseHandle, Handle},
+    Key, Value,
+};
+
+#[derive(Debug, Clone)]
+pub struct LfuConfig {
+    /// `window` capacity ratio of the total cache capacity.
+    ///
+    /// Must be in (0, 1).
+    ///
+    /// Must guarantee `window_capacity_ratio + protected_capacity_ratio < 1`.
+    pub window_capacity_ratio: f64,
+    /// `protected` capacity ratio of the total cache capacity.
+    ///
+    /// Must be in (0, 1).
+    ///
+    /// Must guarantee `window_capacity_ratio + protected_capacity_ratio < 1`.
+    pub protected_capacity_ratio: f64,
+
+    pub cmsketch_eps: f64,
+    pub cmsketch_confidence: f64,
+}
+
+pub type LfuContext = ();
+
+#[derive(Debug, PartialEq, Eq)]
+enum Queue {
+    None,
+    Window,
+    Probation,
+    Protected,
+}
+
+pub struct LfuHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    link: DlistLink,
+    base: BaseHandle<K, V, LfuContext>,
+    queue: Queue,
+}
+
+impl<K, V> Debug for LfuHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LfuHandle").finish()
+    }
+}
+
+intrusive_adapter! { LfuHandleDlistAdapter<K, V> = NonNull<LfuHandle<K, V>>: LfuHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+
+impl<K, V> Handle for LfuHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Context = LfuContext;
+
+    fn new() -> Self {
+        Self {
+            link: DlistLink::default(),
+            base: BaseHandle::new(),
+            queue: Queue::None,
+        }
+    }
+
+    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
+        self.base.init(hash, key, value, charge, context)
+    }
+
+    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &self.base
+    }
+
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &mut self.base
+    }
+}
+
+unsafe impl<K, V> Send for LfuHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+unsafe impl<K, V> Sync for LfuHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+
+/// This implementation is inspired by [Caffeine](https://github.com/ben-manes/caffeine) under Apache License 2.0
+///
+/// A newcoming and hot entry is kept in `window`.
+///
+/// When `window` is full, entries from it will overflow to `probation`.
+///
+/// When a entry in `probation` is accessed, it will be promoted to `protected`.
+///
+/// When `protected` is full, entries from it will overflow to `probation`.
+///
+/// When evicting, the entry with a lower frequency from `window` or `probtion` will be evicted first, then from
+/// `protected`.
+pub struct Lfu<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    window: Dlist<LfuHandleDlistAdapter<K, V>>,
+    probation: Dlist<LfuHandleDlistAdapter<K, V>>,
+    protected: Dlist<LfuHandleDlistAdapter<K, V>>,
+
+    window_charges: usize,
+    probation_charges: usize,
+    protected_charges: usize,
+
+    window_charges_capacity: usize,
+    protected_charges_capacity: usize,
+
+    frequencies: CMSketchU16,
+
+    step: usize,
+    decay: usize,
+}
+
+impl<K, V> Lfu<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn increase_queue_charges(&mut self, handle: &LfuHandle<K, V>) {
+        let charges = handle.base().charge();
+        match handle.queue {
+            Queue::None => unreachable!(),
+            Queue::Window => self.window_charges += charges,
+            Queue::Probation => self.probation_charges += charges,
+            Queue::Protected => self.protected_charges += charges,
+        }
+    }
+
+    fn decrease_queue_charges(&mut self, handle: &LfuHandle<K, V>) {
+        let charges = handle.base().charge();
+        match handle.queue {
+            Queue::None => unreachable!(),
+            Queue::Window => self.window_charges -= charges,
+            Queue::Probation => self.probation_charges -= charges,
+            Queue::Protected => self.protected_charges -= charges,
+        }
+    }
+
+    fn update_frequencies(&mut self, hash: u64) {
+        self.frequencies.inc(hash);
+        self.step += 1;
+        if self.step >= self.decay {
+            self.step >>= 1;
+            self.frequencies.halve();
+        }
+    }
+}
+
+impl<K, V> Eviction for Lfu<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Handle = LfuHandle<K, V>;
+    type Config = LfuConfig;
+
+    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
+    where
+        Self: Sized,
+    {
+        assert!(
+            config.window_capacity_ratio > 0.0 && config.window_capacity_ratio < 1.0,
+            "window_capacity_ratio must be in (0, 1), given: {}",
+            config.window_capacity_ratio
+        );
+
+        assert!(
+            config.protected_capacity_ratio > 0.0 && config.protected_capacity_ratio < 1.0,
+            "protected_capacity_ratio must be in (0, 1), given: {}",
+            config.protected_capacity_ratio
+        );
+
+        assert!(
+            config.window_capacity_ratio + config.protected_capacity_ratio < 1.0,
+            "must guarantee: window_capacity_ratio + protected_capacity_ratio < 1, given: {}",
+            config.window_capacity_ratio + config.protected_capacity_ratio
+        );
+
+        let window_charges_capacity = (capacity as f64 * config.window_capacity_ratio) as usize;
+        let protected_charges_capacity = (capacity as f64 * config.protected_capacity_ratio) as usize;
+        let frequencies = CMSketchU16::new(config.cmsketch_eps, config.cmsketch_confidence);
+        let decay = frequencies.width();
+
+        Self {
+            window: Dlist::new(),
+            probation: Dlist::new(),
+            protected: Dlist::new(),
+            window_charges: 0,
+            probation_charges: 0,
+            protected_charges: 0,
+            window_charges_capacity,
+            protected_charges_capacity,
+            frequencies,
+            step: 0,
+            decay,
+        }
+    }
+
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+
+        debug_assert!(!handle.link.is_linked());
+        debug_assert!(!handle.base().is_in_eviction());
+        debug_assert_eq!(handle.queue, Queue::None);
+
+        self.window.push_back(ptr);
+        handle.base_mut().set_in_eviction(true);
+        handle.queue = Queue::Window;
+
+        self.increase_queue_charges(handle);
+        self.update_frequencies(handle.base().hash());
+
+        // If `window` charges exceeds the capacity, overflow entry from `window` to `probation`.
+        while self.window_charges > self.window_charges_capacity {
+            debug_assert!(!self.window.is_empty());
+            let mut ptr = self.window.pop_front().unwrap_unchecked();
+            let handle = ptr.as_mut();
+            self.decrease_queue_charges(handle);
+            handle.queue = Queue::Probation;
+            self.increase_queue_charges(handle);
+            self.probation.push_back(ptr);
+        }
+    }
+
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+        // Compare the frequency of the front element of `window` and `probation` queue, and evict the lower one.
+        // If both `window` and `probation` are empty, try evict from `protected`.
+        let mut ptr = match (self.window.front(), self.probation.front()) {
+            (None, None) => None,
+            (None, Some(_)) => self.probation.pop_front(),
+            (Some(_), None) => self.window.pop_front(),
+            (Some(window), Some(probation)) => {
+                if self.frequencies.estimate(window.base().hash()) < self.frequencies.estimate(probation.base().hash())
+                {
+                    self.window.pop_front()
+
+                    // TODO(MrCroxx): Rotate probation to prevent a high frequency but cold head holds back promotion
+                    // too long like CacheLib does?
+                } else {
+                    self.probation.pop_front()
+                }
+            }
+        }
+        .or_else(|| self.protected.pop_front())?;
+
+        let handle = ptr.as_mut();
+
+        debug_assert!(!handle.link.is_linked());
+        debug_assert!(handle.base().is_in_eviction());
+        debug_assert_ne!(handle.queue, Queue::None);
+
+        self.decrease_queue_charges(handle);
+        handle.queue = Queue::None;
+        handle.base_mut().set_in_eviction(false);
+
+        Some(ptr)
+    }
+
+    unsafe fn reinsert(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+
+        match handle.queue {
+            Queue::None => {
+                debug_assert!(!handle.link.is_linked());
+                debug_assert!(!handle.base().is_in_eviction());
+                self.push(ptr);
+                debug_assert!(handle.link.is_linked());
+                debug_assert!(handle.base().is_in_eviction());
+            }
+            Queue::Window => {
+                // Move to MRU position of `window`.
+                debug_assert!(handle.link.is_linked());
+                debug_assert!(handle.base().is_in_eviction());
+                self.window.remove_raw(handle.link.raw());
+                self.window.push_back(ptr);
+            }
+            Queue::Probation => {
+                // Promote to MRU position of `protected`.
+                debug_assert!(handle.link.is_linked());
+                debug_assert!(handle.base().is_in_eviction());
+                self.probation.remove_raw(handle.link.raw());
+                self.decrease_queue_charges(handle);
+                handle.queue = Queue::Protected;
+                self.increase_queue_charges(handle);
+                self.protected.push_back(ptr);
+
+                // If `protected` charges exceeds the capacity, overflow entry from `protected` to `probation`.
+                while self.protected_charges > self.protected_charges_capacity {
+                    debug_assert!(!self.protected.is_empty());
+                    let mut ptr = self.protected.pop_front().unwrap_unchecked();
+                    let handle = ptr.as_mut();
+                    self.decrease_queue_charges(handle);
+                    handle.queue = Queue::Probation;
+                    self.increase_queue_charges(handle);
+                    self.probation.push_back(ptr);
+                }
+            }
+            Queue::Protected => {
+                // Move to MRU position of `protected`.
+                debug_assert!(handle.link.is_linked());
+                debug_assert!(handle.base().is_in_eviction());
+                self.protected.remove_raw(handle.link.raw());
+                self.protected.push_back(ptr);
+            }
+        }
+    }
+
+    unsafe fn access(&mut self, ptr: NonNull<Self::Handle>) {
+        self.update_frequencies(ptr.as_ref().base().hash());
+    }
+
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+
+        debug_assert!(handle.link.is_linked());
+        debug_assert!(handle.base().is_in_eviction());
+        debug_assert_ne!(handle.queue, Queue::None);
+
+        match handle.queue {
+            Queue::None => unreachable!(),
+            Queue::Window => self.window.remove_raw(handle.link.raw()),
+            Queue::Probation => self.probation.remove_raw(handle.link.raw()),
+            Queue::Protected => self.protected.remove_raw(handle.link.raw()),
+        };
+
+        debug_assert!(!handle.link.is_linked());
+
+        self.decrease_queue_charges(handle);
+        handle.queue = Queue::None;
+        handle.base_mut().set_in_eviction(false);
+    }
+
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+        let mut res = Vec::with_capacity(self.len());
+
+        while !self.is_empty() {
+            let ptr = self.pop().unwrap_unchecked();
+            debug_assert!(!ptr.as_ref().base().is_in_eviction());
+            debug_assert!(!ptr.as_ref().link.is_linked());
+            debug_assert_eq!(ptr.as_ref().queue, Queue::None);
+            res.push(ptr);
+        }
+
+        res
+    }
+
+    unsafe fn len(&self) -> usize {
+        self.window.len() + self.probation.len() + self.protected.len()
+    }
+
+    unsafe fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+unsafe impl<K, V> Send for Lfu<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+unsafe impl<K, V> Sync for Lfu<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+
+#[cfg(test)]
+mod tests {
+
+    use itertools::Itertools;
+
+    use super::*;
+    use crate::eviction::test_utils::TestEviction;
+
+    impl<K, V> TestEviction for Lfu<K, V>
+    where
+        K: Key + Clone,
+        V: Value + Clone,
+    {
+        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+            self.window
+                .iter()
+                .chain(self.probation.iter())
+                .chain(self.protected.iter())
+                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .collect_vec()
+        }
+    }
+
+    type TestLfu = Lfu<u64, u64>;
+    type TestLfuHandle = LfuHandle<u64, u64>;
+
+    unsafe fn assert_test_lfu(
+        lfu: &TestLfu,
+        len: usize,
+        window: usize,
+        probation: usize,
+        protected: usize,
+        entries: Vec<u64>,
+    ) {
+        assert_eq!(lfu.len(), len);
+        assert_eq!(lfu.window.len(), window);
+        assert_eq!(lfu.probation.len(), probation);
+        assert_eq!(lfu.protected.len(), protected);
+        assert_eq!(lfu.window_charges, window);
+        assert_eq!(lfu.probation_charges, probation);
+        assert_eq!(lfu.protected_charges, protected);
+        let es = lfu
+            .dump()
+            .into_iter()
+            .map(|(k, v)| {
+                assert_eq!(k, v);
+                k
+            })
+            .collect_vec();
+        assert_eq!(es, entries);
+    }
+
+    fn assert_min_frequency(lfu: &TestLfu, hash: u64, count: usize) {
+        let freq = lfu.frequencies.estimate(hash);
+        assert!(freq >= count as u16, "assert {freq} >= {count} failed for {hash}");
+    }
+
+    #[test]
+    fn test_lfu() {
+        unsafe {
+            let ptrs = (0..100)
+                .map(|i| {
+                    let mut handle = Box::new(TestLfuHandle::new());
+                    handle.init(i, i, i, 1, ());
+                    NonNull::new_unchecked(Box::into_raw(handle))
+                })
+                .collect_vec();
+
+            // window: 2, probation: 2, protected: 6
+            let config = LfuConfig {
+                window_capacity_ratio: 0.2,
+                protected_capacity_ratio: 0.6,
+                cmsketch_eps: 0.01,
+                cmsketch_confidence: 0.95,
+            };
+            let mut lfu = TestLfu::new(10, &config);
+
+            assert_eq!(lfu.window_charges_capacity, 2);
+            assert_eq!(lfu.protected_charges_capacity, 6);
+
+            lfu.push(ptrs[0]);
+            lfu.push(ptrs[1]);
+            assert_test_lfu(&lfu, 2, 2, 0, 0, vec![0, 1]);
+
+            lfu.push(ptrs[2]);
+            lfu.push(ptrs[3]);
+            assert_test_lfu(&lfu, 4, 2, 2, 0, vec![2, 3, 0, 1]);
+
+            (4..10).for_each(|i| lfu.push(ptrs[i]));
+            assert_test_lfu(&lfu, 10, 2, 8, 0, vec![8, 9, 0, 1, 2, 3, 4, 5, 6, 7]);
+
+            (0..10).for_each(|i| assert_min_frequency(&lfu, i, 1));
+
+            // [8, 9] [1, 2, 3, 4, 5, 6, 7]
+            let p0 = lfu.pop().unwrap();
+            assert_eq!(p0, ptrs[0]);
+
+            // [9, 0] [1, 2, 3, 4, 5, 6, 7, 8]
+            lfu.reinsert(p0);
+            assert_test_lfu(&lfu, 10, 2, 8, 0, vec![9, 0, 1, 2, 3, 4, 5, 6, 7, 8]);
+
+            // [0, 9] [1, 2, 3, 4, 5, 6, 7, 8]
+            lfu.reinsert(ptrs[9]);
+            assert_test_lfu(&lfu, 10, 2, 8, 0, vec![0, 9, 1, 2, 3, 4, 5, 6, 7, 8]);
+
+            // [0, 9] [1, 2, 7, 8] [3, 4, 5, 6]
+            (3..7).for_each(|i| lfu.reinsert(ptrs[i]));
+            assert_test_lfu(&lfu, 10, 2, 4, 4, vec![0, 9, 1, 2, 7, 8, 3, 4, 5, 6]);
+
+            // [0, 9] [1, 2, 7, 8] [5, 6, 3, 4]
+            (3..5).for_each(|i| lfu.reinsert(ptrs[i]));
+            assert_test_lfu(&lfu, 10, 2, 4, 4, vec![0, 9, 1, 2, 7, 8, 5, 6, 3, 4]);
+
+            // [0, 9] [5, 6] [3, 4, 1, 2, 7, 8]
+            [1, 2, 7, 8].into_iter().for_each(|i| lfu.reinsert(ptrs[i]));
+            assert_test_lfu(&lfu, 10, 2, 2, 6, vec![0, 9, 5, 6, 3, 4, 1, 2, 7, 8]);
+
+            // [0, 9] [6] [3, 4, 1, 2, 7, 8]
+            let p5 = lfu.pop().unwrap();
+            assert_eq!(p5, ptrs[5]);
+            assert_test_lfu(&lfu, 9, 2, 1, 6, vec![0, 9, 6, 3, 4, 1, 2, 7, 8]);
+
+            (10..13).for_each(|i| lfu.push(ptrs[i]));
+
+            // [11, 12] [6, 0, 9, 10] [3, 4, 1, 2, 7, 8]
+            assert_test_lfu(&lfu, 12, 2, 4, 6, vec![11, 12, 6, 0, 9, 10, 3, 4, 1, 2, 7, 8]);
+            (1..13).for_each(|i| assert_min_frequency(&lfu, i, 0));
+            lfu.access(ptrs[0]);
+            assert_min_frequency(&lfu, 0, 2);
+
+            // evict 11 because freq(11) < freq(0)
+            // [12] [0, 9, 10] [3, 4, 1, 2, 7, 8]
+            let p6 = lfu.pop().unwrap();
+            let p11 = lfu.pop().unwrap();
+            assert_eq!(p6, ptrs[6]);
+            assert_eq!(p11, ptrs[11]);
+            assert_test_lfu(&lfu, 10, 1, 3, 6, vec![12, 0, 9, 10, 3, 4, 1, 2, 7, 8]);
+
+            assert_eq!(
+                lfu.clear(),
+                [12, 0, 9, 10, 3, 4, 1, 2, 7, 8]
+                    .into_iter()
+                    .map(|i| ptrs[i])
+                    .collect_vec()
+            );
+
+            for ptr in ptrs {
+                let _ = Box::from_raw(ptr.as_ptr());
+            }
+        }
+    }
+}
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index 3a78e74e..57138cae 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -102,6 +102,7 @@ pub trait Eviction: Send + Sync + 'static {
 }
 
 pub mod fifo;
+pub mod lfu;
 pub mod lru;
 
 #[cfg(test)]

From 7cb52d5b17a59eba4cf58567f42a7dd2a5f5c3be Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Mar 2024 11:55:02 +0800
Subject: [PATCH 223/261] refactor: use generic type instead of associated type
 for listener

* refactor: make context listener's generic type and alias for simplicity

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: remove context from listener, because cannot impl trait alias

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: restore context for listener, make user suffer

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs                  | 44 ++++++++++++----------
 foyer-memory/src/handle.rs                 |  4 +-
 foyer-memory/src/lib.rs                    | 20 +++++-----
 foyer-memory/src/{event.rs => listener.rs} | 31 +++++++--------
 4 files changed, 51 insertions(+), 48 deletions(-)
 rename foyer-memory/src/{event.rs => listener.rs} (60%)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 2721ce39..ec4afaf2 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -32,7 +32,6 @@ use parking_lot::Mutex;
 use tokio::{sync::oneshot, task::JoinHandle};
 
 use crate::{
-    event::{CacheEventListener, DefaultCacheEventListener},
     eviction::{
         fifo::{Fifo, FifoContext, FifoHandle},
         lfu::{Lfu, LfuContext, LfuHandle},
@@ -41,6 +40,7 @@ use crate::{
     },
     handle::Handle,
     indexer::{HashTableIndexer, Indexer},
+    listener::{CacheEventListener, DefaultCacheEventListener},
     metrics::Metrics,
     Key, Value,
 };
@@ -60,7 +60,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     indexer: I,
@@ -81,7 +81,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn new(
@@ -291,7 +291,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn drop(&mut self) {
@@ -302,7 +302,7 @@ where
 pub struct CacheConfig<E, L, S = RandomState>
 where
     E: Eviction,
-    L: CacheEventListener,
+    L: CacheEventListener<<E::Handle as Handle>::Key, <E::Handle as Handle>::Value, <E::Handle as Handle>::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub capacity: usize,
@@ -321,7 +321,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
 {
@@ -338,7 +338,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
 {
@@ -354,7 +354,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error + From<oneshot::error::RecvError>,
 {
@@ -381,7 +381,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     shards: Vec<Mutex<CacheShard<K, V, H, E, I, L, S>>>,
@@ -401,7 +401,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub fn new(config: CacheConfig<E, L, S>) -> Self {
@@ -543,7 +543,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> Entry<K, V, H, E, I, L, S, ER>
@@ -610,7 +610,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     cache: Arc<Cache<K, V, H, E, I, L, S>>,
@@ -624,7 +624,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub fn key(&self) -> &H::Key {
@@ -655,7 +655,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn clone(&self) -> Self {
@@ -681,7 +681,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn drop(&mut self) {
@@ -696,7 +696,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
     type Target = V;
@@ -713,7 +713,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
 }
@@ -724,7 +724,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<Key = K, Value = V, Context = H::Context>,
+    L: CacheEventListener<K, V, H::Context>,
     S: BuildHasher + Send + Sync + 'static,
 {
 }
@@ -756,8 +756,12 @@ mod tests {
 
     use super::*;
     use crate::{
-        event::DefaultCacheEventListener,
-        eviction::{fifo::FifoConfig, lru::LruConfig, test_utils::TestEviction},
+        eviction::{
+            fifo::{FifoConfig, FifoHandle},
+            lru::LruConfig,
+            test_utils::TestEviction,
+        },
+        listener::DefaultCacheEventListener,
     };
 
     fn is_send_sync_static<T: Send + Sync + 'static>() {}
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
index f1fea9d3..f1b2d0f4 100644
--- a/foyer-memory/src/handle.rs
+++ b/foyer-memory/src/handle.rs
@@ -14,7 +14,7 @@
 
 use bitflags::bitflags;
 
-use crate::{Key, Value};
+use crate::{Context, Key, Value};
 
 bitflags! {
     #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -27,7 +27,7 @@ bitflags! {
 pub trait Handle: Send + Sync + 'static {
     type Key: Key;
     type Value: Value;
-    type Context: Default;
+    type Context: Context;
 
     fn new() -> Self;
     fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context);
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 0add22eb..14e80881 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -65,17 +65,19 @@
 //! The handle that does not appear in either the indexer or the eviction container, and has no external owner, will be
 //! destroyed.
 
+#![feature(trait_alias)]
+
+pub trait Key: Send + Sync + 'static + std::hash::Hash + Eq + Ord {}
+pub trait Value: Send + Sync + 'static {}
+pub trait Context: Send + Sync + 'static + Default {}
+
+impl<T: Send + Sync + 'static + std::hash::Hash + Eq + Ord> Key for T {}
+impl<T: Send + Sync + 'static> Value for T {}
+impl<T: Send + Sync + 'static + Default> Context for T {}
+
 pub mod cache;
-pub mod event;
 pub mod eviction;
 pub mod handle;
 pub mod indexer;
+pub mod listener;
 pub mod metrics;
-
-use std::hash::Hash;
-
-pub trait Key: Send + Sync + 'static + Hash + Eq + Ord {}
-pub trait Value: Send + Sync + 'static {}
-
-impl<T: Send + Sync + 'static + Hash + Eq + Ord> Key for T {}
-impl<T: Send + Sync + 'static> Value for T {}
diff --git a/foyer-memory/src/event.rs b/foyer-memory/src/listener.rs
similarity index 60%
rename from foyer-memory/src/event.rs
rename to foyer-memory/src/listener.rs
index c5b9c348..ec088e88 100644
--- a/foyer-memory/src/event.rs
+++ b/foyer-memory/src/listener.rs
@@ -14,45 +14,42 @@
 
 use std::marker::PhantomData;
 
-use crate::{Key, Value};
-
-pub trait CacheEventListener: Send + Sync + 'static {
-    type Key: Key;
-    type Value: Value;
-    type Context: Send + Sync + 'static;
+use crate::{Context, Key, Value};
 
+pub trait CacheEventListener<K, V, C>: Send + Sync + 'static
+where
+    K: Key,
+    V: Value,
+    C: Context,
+{
     /// The function is called when an entry is released by the cache and all external users.
     ///
-    /// The arguments includes the key, value and context with ownership.
-    fn on_release(&self, key: Self::Key, value: Self::Value, context: Self::Context, charges: usize);
+    /// The arguments includes the key and value with ownership.
+    fn on_release(&self, key: K, value: V, context: C, charges: usize);
 }
 
 pub struct DefaultCacheEventListener<K, V, C>(PhantomData<(K, V, C)>)
 where
     K: Key,
     V: Value,
-    C: Send + Sync + 'static;
+    C: Context;
 
 impl<K, V, C> Default for DefaultCacheEventListener<K, V, C>
 where
     K: Key,
     V: Value,
-    C: Send + Sync + 'static,
+    C: Context,
 {
     fn default() -> Self {
         Self(Default::default())
     }
 }
 
-impl<K, V, C> CacheEventListener for DefaultCacheEventListener<K, V, C>
+impl<K, V, C> CacheEventListener<K, V, C> for DefaultCacheEventListener<K, V, C>
 where
     K: Key,
     V: Value,
-    C: Send + Sync + 'static,
+    C: Context,
 {
-    type Key = K;
-    type Value = V;
-    type Context = C;
-
-    fn on_release(&self, _key: Self::Key, _value: Self::Value, _context: Self::Context, _charges: usize) {}
+    fn on_release(&self, _key: K, _value: V, _context: C, _charges: usize) {}
 }

From fd86347c56f32623ca1d5820e373ced89e85abfc Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Mar 2024 13:20:45 +0800
Subject: [PATCH 224/261] refactor: use prelude mod to restrict visibility
 (#285)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs   | 34 ++--------------------
 foyer-memory/src/indexer.rs |  1 -
 foyer-memory/src/lib.rs     | 15 ++++++----
 foyer-memory/src/prelude.rs | 57 +++++++++++++++++++++++++++++++++++++
 4 files changed, 68 insertions(+), 39 deletions(-)
 create mode 100644 foyer-memory/src/prelude.rs

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index ec4afaf2..4aae4161 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -32,17 +32,7 @@ use parking_lot::Mutex;
 use tokio::{sync::oneshot, task::JoinHandle};
 
 use crate::{
-    eviction::{
-        fifo::{Fifo, FifoContext, FifoHandle},
-        lfu::{Lfu, LfuContext, LfuHandle},
-        lru::{Lru, LruContext, LruHandle},
-        Eviction,
-    },
-    handle::Handle,
-    indexer::{HashTableIndexer, Indexer},
-    listener::{CacheEventListener, DefaultCacheEventListener},
-    metrics::Metrics,
-    Key, Value,
+    eviction::Eviction, handle::Handle, indexer::Indexer, listener::CacheEventListener, metrics::Metrics, Key, Value,
 };
 
 struct CacheContext<T, L> {
@@ -729,27 +719,6 @@ where
 {
 }
 
-pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
-    Cache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
-pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
-    CacheConfig<Fifo<K, V>, L, S>;
-pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
-    CacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
-
-pub type LruCache<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
-    Cache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
-pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
-    CacheConfig<Lru<K, V>, L, S>;
-pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
-    CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
-
-pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
-    Cache<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
-pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
-    CacheConfig<Lfu<K, V>, L, S>;
-pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
-    CacheEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
-
 #[cfg(test)]
 mod tests {
     use rand::{rngs::SmallRng, RngCore, SeedableRng};
@@ -762,6 +731,7 @@ mod tests {
             test_utils::TestEviction,
         },
         listener::DefaultCacheEventListener,
+        prelude::{FifoCache, FifoCacheConfig, FifoCacheEntry, LruCache, LruCacheConfig, LruCacheEntry},
     };
 
     fn is_send_sync_static<T: Send + Sync + 'static>() {}
diff --git a/foyer-memory/src/indexer.rs b/foyer-memory/src/indexer.rs
index fd49557c..df9fd630 100644
--- a/foyer-memory/src/indexer.rs
+++ b/foyer-memory/src/indexer.rs
@@ -18,7 +18,6 @@ use hashbrown::hash_table::{Entry as HashTableEntry, HashTable};
 
 use crate::{handle::Handle, Key};
 
-#[expect(clippy::missing_safety_doc)]
 pub trait Indexer: Send + Sync + 'static {
     type Key: Key;
     type Handle: Handle<Key = Self::Key>;
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 14e80881..c3a4ad92 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -75,9 +75,12 @@ impl<T: Send + Sync + 'static + std::hash::Hash + Eq + Ord> Key for T {}
 impl<T: Send + Sync + 'static> Value for T {}
 impl<T: Send + Sync + 'static + Default> Context for T {}
 
-pub mod cache;
-pub mod eviction;
-pub mod handle;
-pub mod indexer;
-pub mod listener;
-pub mod metrics;
+mod cache;
+mod eviction;
+mod handle;
+mod indexer;
+mod listener;
+mod metrics;
+mod prelude;
+
+pub use prelude::*;
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
new file mode 100644
index 00000000..a99f0c26
--- /dev/null
+++ b/foyer-memory/src/prelude.rs
@@ -0,0 +1,57 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use ahash::RandomState;
+
+use crate::{
+    cache::{Cache, CacheConfig, CacheEntry},
+    eviction::{
+        fifo::{Fifo, FifoHandle},
+        lfu::{Lfu, LfuHandle},
+        lru::{Lru, LruHandle},
+    },
+    indexer::HashTableIndexer,
+    listener::DefaultCacheEventListener,
+};
+
+pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+    Cache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+    CacheConfig<Fifo<K, V>, L, S>;
+pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+    CacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+
+pub type LruCache<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+    Cache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+    CacheConfig<Lru<K, V>, L, S>;
+pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+    CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+
+pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+    Cache<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+    CacheConfig<Lfu<K, V>, L, S>;
+pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+    CacheEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+
+pub use crate::{
+    cache::Entry,
+    eviction::{
+        fifo::{FifoConfig, FifoContext},
+        lfu::{LfuConfig, LfuContext},
+        lru::{LruConfig, LruContext},
+    },
+    listener::CacheEventListener,
+};

From 960ddddcdfa6644d0b33a25456797bf5064ce00b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Mar 2024 14:11:23 +0800
Subject: [PATCH 225/261] chore: upgrade  dependencies, fix target auto detect
 (#286)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml                          |  3 ---
 foyer-experimental-bench/Cargo.toml | 14 ++++++++------
 foyer-storage-bench/Cargo.toml      | 12 ++++++------
 foyer-storage/Cargo.toml            |  3 +--
 4 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index ba63fa62..e91dfe93 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -23,8 +23,5 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
     "fs",
 ] }
 
-[patch.crates-io]
-# cmsketch = { path = "../cmsketch-rs" }
-
 [profile.release]
 debug = "full"
diff --git a/foyer-experimental-bench/Cargo.toml b/foyer-experimental-bench/Cargo.toml
index 2c507afc..f8cc3b81 100644
--- a/foyer-experimental-bench/Cargo.toml
+++ b/foyer-experimental-bench/Cargo.toml
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+autobenches = false
 
 [package.metadata.cargo-udeps.ignore]
 normal = ["foyer-workspace-hack"]
@@ -35,11 +36,11 @@ hyper-util = { version = "0.1", features = [
 ] }
 itertools = "0.12"
 libc = "0.2"
-nix = { version = "0.27", features = ["fs", "mman"] }
-opentelemetry = { version = "0.21", optional = true }
-opentelemetry-otlp = { version = "0.14.0", optional = true }
-opentelemetry-semantic-conventions = { version = "0.13", optional = true }
-opentelemetry_sdk = { version = "0.21", features = [
+nix = { version = "0.28", features = ["fs", "mman"] }
+opentelemetry = { version = "0.22", optional = true }
+opentelemetry-otlp = { version = "0.15.0", optional = true }
+opentelemetry-semantic-conventions = { version = "0.14", optional = true }
+opentelemetry_sdk = { version = "0.22", features = [
     "rt-tokio",
     "trace",
 ], optional = true }
@@ -48,7 +49,7 @@ prometheus = "0.13"
 rand = "0.8.5"
 tokio = { workspace = true }
 tracing = "0.1"
-tracing-opentelemetry = { version = "0.22", optional = true }
+tracing-opentelemetry = { version = "0.23", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [features]
@@ -65,3 +66,4 @@ trace = [
 [[bin]]
 name = "wal-bench"
 path = "benches/wal-bench.rs"
+bench = false
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index e1923ec6..b0dc51b6 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -34,11 +34,11 @@ hyper-util = { version = "0.1", features = [
 ] }
 itertools = "0.12"
 libc = "0.2"
-nix = { version = "0.27", features = ["fs", "mman"] }
-opentelemetry = { version = "0.21", optional = true }
-opentelemetry-otlp = { version = "0.14.0", optional = true }
-opentelemetry-semantic-conventions = { version = "0.13", optional = true }
-opentelemetry_sdk = { version = "0.21", features = [
+nix = { version = "0.28", features = ["fs", "mman"] }
+opentelemetry = { version = "0.22", optional = true }
+opentelemetry-otlp = { version = "0.15.0", optional = true }
+opentelemetry-semantic-conventions = { version = "0.14", optional = true }
+opentelemetry_sdk = { version = "0.22", features = [
     "rt-tokio",
     "trace",
 ], optional = true }
@@ -47,7 +47,7 @@ prometheus = "0.13"
 rand = "0.8.5"
 tokio = { workspace = true }
 tracing = "0.1"
-tracing-opentelemetry = { version = "0.22", optional = true }
+tracing-opentelemetry = { version = "0.23", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zipf = "7"
 
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 1b370d80..69b91a79 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -17,7 +17,6 @@ anyhow = "1.0"
 bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
-cmsketch = "0.1"
 foyer-common = { version = "0.3", path = "../foyer-common" }
 foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
@@ -26,7 +25,7 @@ itertools = "0.12"
 libc = "0.2"
 lz4 = "1.24"
 memoffset = "0.9"
-nix = { version = "0.27", features = ["fs", "mman", "uio"] }
+nix = { version = "0.28", features = ["fs", "mman", "uio"] }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 prometheus = "0.13"

From fe10803de407d59c3ab2b02e5d0bbef1d3f57948 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 11 Mar 2024 17:14:31 +0800
Subject: [PATCH 226/261] refactor: introduce a unified cache context (#287)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs         | 89 ++++++++++++++++---------------
 foyer-memory/src/context.rs       | 30 +++++++++++
 foyer-memory/src/eviction/fifo.rs | 19 +++++--
 foyer-memory/src/eviction/lfu.rs  | 19 +++++--
 foyer-memory/src/eviction/lru.rs  | 20 +++++--
 foyer-memory/src/handle.rs        |  2 +-
 foyer-memory/src/lib.rs           |  3 +-
 foyer-memory/src/listener.rs      | 20 +++----
 foyer-memory/src/prelude.rs       | 22 ++++----
 9 files changed, 143 insertions(+), 81 deletions(-)
 create mode 100644 foyer-memory/src/context.rs

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 4aae4161..d560bb0c 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -32,10 +32,11 @@ use parking_lot::Mutex;
 use tokio::{sync::oneshot, task::JoinHandle};
 
 use crate::{
-    eviction::Eviction, handle::Handle, indexer::Indexer, listener::CacheEventListener, metrics::Metrics, Key, Value,
+    eviction::Eviction, handle::Handle, indexer::Indexer, listener::CacheEventListener, metrics::Metrics, CacheContext,
+    Key, Value,
 };
 
-struct CacheContext<T, L> {
+struct CacheSharedState<T, L> {
     metrics: Metrics,
     /// The object pool to avoid frequent handle allocating, shared by all shards.
     object_pool: ArrayQueue<Box<T>>,
@@ -50,7 +51,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     indexer: I,
@@ -61,7 +62,7 @@ where
 
     waiters: HashMap<K, Vec<oneshot::Sender<CacheEntry<K, V, H, E, I, L, S>>>>,
 
-    context: Arc<CacheContext<H, L>>,
+    state: Arc<CacheSharedState<H, L>>,
 }
 
 impl<K, V, H, E, I, L, S> CacheShard<K, V, H, E, I, L, S>
@@ -71,14 +72,14 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn new(
         capacity: usize,
         eviction_config: &E::Config,
         usage: Arc<AtomicUsize>,
-        context: Arc<CacheContext<H, L>>,
+        context: Arc<CacheSharedState<H, L>>,
     ) -> Self {
         let indexer = I::new();
         let eviction = unsafe { E::new(capacity, eviction_config) };
@@ -89,7 +90,7 @@ where
             capacity,
             usage,
             waiters,
-            context,
+            state: context,
         }
     }
 
@@ -103,7 +104,7 @@ where
         context: H::Context,
         last_reference_entries: &mut Vec<(K, V, H::Context, usize)>,
     ) -> NonNull<H> {
-        let mut handle = self.context.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
+        let mut handle = self.state.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
         handle.init(hash, key, value, charge, context);
         let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
 
@@ -111,7 +112,7 @@ where
 
         debug_assert!(!ptr.as_ref().base().is_in_indexer());
         if let Some(old) = self.indexer.insert(ptr) {
-            self.context.metrics.replace.fetch_add(1, Ordering::Relaxed);
+            self.state.metrics.replace.fetch_add(1, Ordering::Relaxed);
 
             debug_assert!(!old.as_ref().base().is_in_indexer());
             if old.as_ref().base().is_in_eviction() {
@@ -123,7 +124,7 @@ where
                 last_reference_entries.push(entry);
             }
         } else {
-            self.context.metrics.insert.fetch_add(1, Ordering::Relaxed);
+            self.state.metrics.insert.fetch_add(1, Ordering::Relaxed);
         }
         self.eviction.push(ptr);
 
@@ -139,11 +140,11 @@ where
     unsafe fn get(&mut self, hash: u64, key: &K) -> Option<NonNull<H>> {
         let mut ptr = match self.indexer.get(hash, key) {
             Some(ptr) => {
-                self.context.metrics.hit.fetch_add(1, Ordering::Relaxed);
+                self.state.metrics.hit.fetch_add(1, Ordering::Relaxed);
                 ptr
             }
             None => {
-                self.context.metrics.miss.fetch_add(1, Ordering::Relaxed);
+                self.state.metrics.miss.fetch_add(1, Ordering::Relaxed);
                 return None;
             }
         };
@@ -161,7 +162,7 @@ where
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
     unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V, H::Context, usize)> {
         let ptr = self.indexer.remove(hash, key)?;
-        self.context.metrics.remove.fetch_add(1, Ordering::Relaxed);
+        self.state.metrics.remove.fetch_add(1, Ordering::Relaxed);
         if ptr.as_ref().base().is_in_eviction() {
             self.eviction.remove(ptr);
         }
@@ -184,7 +185,7 @@ where
             debug_assert!((&eptrs - &ptrs).is_empty());
         }
 
-        self.context.metrics.remove.fetch_add(ptrs.len(), Ordering::Relaxed);
+        self.state.metrics.remove.fetch_add(ptrs.len(), Ordering::Relaxed);
 
         // The handles in the indexer covers the handles in the eviction container.
         // So only the handles drained from the indexer need to be released.
@@ -200,7 +201,7 @@ where
         while self.usage.load(Ordering::Relaxed) + charge > self.capacity
             && let Some(evicted) = self.eviction.pop()
         {
-            self.context.metrics.evict.fetch_add(1, Ordering::Relaxed);
+            self.state.metrics.evict.fetch_add(1, Ordering::Relaxed);
             let base = evicted.as_ref().base();
             debug_assert!(base.is_in_indexer());
             debug_assert!(!base.is_in_eviction());
@@ -244,7 +245,7 @@ where
                 self.eviction.reinsert(ptr);
                 if ptr.as_ref().base().is_in_eviction() {
                     if was_in_eviction {
-                        self.context.metrics.reinsert.fetch_add(1, Ordering::Relaxed);
+                        self.state.metrics.reinsert.fetch_add(1, Ordering::Relaxed);
                     }
                     return None;
                 }
@@ -262,13 +263,13 @@ where
         debug_assert!(!base.is_in_eviction());
         debug_assert!(!base.has_refs());
 
-        self.context.metrics.release.fetch_add(1, Ordering::Relaxed);
+        self.state.metrics.release.fetch_add(1, Ordering::Relaxed);
 
         self.usage.fetch_sub(base.charge(), Ordering::Relaxed);
         let entry = base.take();
 
         let handle = Box::from_raw(ptr.as_ptr());
-        let _ = self.context.object_pool.push(handle);
+        let _ = self.state.object_pool.push(handle);
 
         Some(entry)
     }
@@ -281,7 +282,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn drop(&mut self) {
@@ -292,7 +293,7 @@ where
 pub struct CacheConfig<E, L, S = RandomState>
 where
     E: Eviction,
-    L: CacheEventListener<<E::Handle as Handle>::Key, <E::Handle as Handle>::Value, <E::Handle as Handle>::Context>,
+    L: CacheEventListener<<E::Handle as Handle>::Key, <E::Handle as Handle>::Value>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub capacity: usize,
@@ -311,7 +312,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
 {
@@ -328,7 +329,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
 {
@@ -344,7 +345,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error + From<oneshot::error::RecvError>,
 {
@@ -371,7 +372,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     shards: Vec<Mutex<CacheShard<K, V, H, E, I, L, S>>>,
@@ -379,7 +380,7 @@ where
     capacity: usize,
     usages: Vec<Arc<AtomicUsize>>,
 
-    context: Arc<CacheContext<H, L>>,
+    context: Arc<CacheSharedState<H, L>>,
 
     hash_builder: S,
 }
@@ -391,12 +392,12 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub fn new(config: CacheConfig<E, L, S>) -> Self {
         let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
-        let context = Arc::new(CacheContext {
+        let context = Arc::new(CacheSharedState {
             metrics: Metrics::default(),
             object_pool: ArrayQueue::new(config.object_pool_capacity),
             listener: config.event_listener,
@@ -420,7 +421,7 @@ where
     }
 
     pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> CacheEntry<K, V, H, E, I, L, S> {
-        self.insert_with_context(key, value, charge, H::Context::default())
+        self.insert_with_context(key, value, charge, CacheContext::default())
     }
 
     pub fn insert_with_context(
@@ -428,7 +429,7 @@ where
         key: K,
         value: V,
         charge: usize,
-        context: H::Context,
+        context: CacheContext,
     ) -> CacheEntry<K, V, H, E, I, L, S> {
         let hash = self.hash_builder.hash_one(&key);
 
@@ -437,7 +438,7 @@ where
         let (entry, waiters) = unsafe {
             let mut shard = self.shards[hash as usize % self.shards.len()].lock();
             let waiters = shard.waiters.remove(&key);
-            let mut ptr = shard.insert(hash, key, value, charge, context, &mut to_deallocate);
+            let mut ptr = shard.insert(hash, key, value, charge, context.into(), &mut to_deallocate);
             if let Some(waiters) = waiters.as_ref() {
                 ptr.as_mut().base_mut().inc_refs_by(waiters.len());
             }
@@ -459,7 +460,7 @@ where
 
         // Do not deallocate data within the lock section.
         for (key, value, context, charges) in to_deallocate {
-            self.context.listener.on_release(key, value, context, charges)
+            self.context.listener.on_release(key, value, context.into(), charges)
         }
 
         entry
@@ -475,7 +476,7 @@ where
 
         // Do not deallocate data within the lock section.
         if let Some((key, value, context, charges)) = entry {
-            self.context.listener.on_release(key, value, context, charges);
+            self.context.listener.on_release(key, value, context.into(), charges);
         }
     }
 
@@ -520,7 +521,7 @@ where
 
         // Do not deallocate data within the lock section.
         if let Some((key, value, context, charges)) = entry {
-            self.context.listener.on_release(key, value, context, charges);
+            self.context.listener.on_release(key, value, context.into(), charges);
         }
     }
 }
@@ -533,7 +534,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> Entry<K, V, H, E, I, L, S, ER>
@@ -573,7 +574,7 @@ where
                         };
 
                         let entry = if let Some(context) = context {
-                            cache.insert_with_context(key, value, charge, context)
+                            cache.insert_with_context(key, value, charge, context.into())
                         } else {
                             cache.insert(key, value, charge)
                         };
@@ -584,8 +585,8 @@ where
                 }
             };
             match entry {
-                Entry::Wait(_) => shard.context.metrics.queue.fetch_add(1, Ordering::Relaxed),
-                Entry::Miss(_) => shard.context.metrics.fetch.fetch_add(1, Ordering::Relaxed),
+                Entry::Wait(_) => shard.state.metrics.queue.fetch_add(1, Ordering::Relaxed),
+                Entry::Miss(_) => shard.state.metrics.fetch.fetch_add(1, Ordering::Relaxed),
                 _ => unreachable!(),
             };
             entry
@@ -600,7 +601,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     cache: Arc<Cache<K, V, H, E, I, L, S>>,
@@ -614,7 +615,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub fn key(&self) -> &H::Key {
@@ -645,7 +646,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn clone(&self) -> Self {
@@ -671,7 +672,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     fn drop(&mut self) {
@@ -686,7 +687,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     type Target = V;
@@ -703,7 +704,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
 }
@@ -714,7 +715,7 @@ where
     H: Handle<Key = K, Value = V>,
     E: Eviction<Handle = H>,
     I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V, H::Context>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
 }
diff --git a/foyer-memory/src/context.rs b/foyer-memory/src/context.rs
new file mode 100644
index 00000000..041fcd78
--- /dev/null
+++ b/foyer-memory/src/context.rs
@@ -0,0 +1,30 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+pub enum CacheContext {
+    /// The default context shared by all eviction container implementations.
+    Default,
+    LruPriorityLow,
+}
+
+impl Default for CacheContext {
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// The overhead of `Context` itself and the conversion should be light.
+pub trait Context: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static {}
+
+impl<T> Context for T where T: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static {}
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index f0180ea5..208e5c06 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -22,10 +22,23 @@ use foyer_intrusive::{
 use crate::{
     eviction::Eviction,
     handle::{BaseHandle, Handle},
-    Key, Value,
+    CacheContext, Key, Value,
 };
 
-pub type FifoContext = ();
+#[derive(Debug, Default)]
+pub struct FifoContext;
+
+impl From<CacheContext> for FifoContext {
+    fn from(_: CacheContext) -> Self {
+        Self
+    }
+}
+
+impl From<FifoContext> for CacheContext {
+    fn from(_: FifoContext) -> Self {
+        CacheContext::Default
+    }
+}
 
 pub struct FifoHandle<K, V>
 where
@@ -182,7 +195,7 @@ pub mod tests {
 
     unsafe fn new_test_fifo_handle_ptr(key: u64, value: u64) -> NonNull<TestFifoHandle> {
         let mut handle = Box::new(TestFifoHandle::new());
-        handle.init(0, key, value, 1, ());
+        handle.init(0, key, value, 1, FifoContext);
         NonNull::new_unchecked(Box::into_raw(handle))
     }
 
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index 89d529f1..dbbfd4d0 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -24,7 +24,7 @@ use foyer_intrusive::{
 use crate::{
     eviction::Eviction,
     handle::{BaseHandle, Handle},
-    Key, Value,
+    CacheContext, Key, Value,
 };
 
 #[derive(Debug, Clone)]
@@ -46,7 +46,20 @@ pub struct LfuConfig {
     pub cmsketch_confidence: f64,
 }
 
-pub type LfuContext = ();
+#[derive(Debug, Default)]
+pub struct LfuContext;
+
+impl From<CacheContext> for LfuContext {
+    fn from(_: CacheContext) -> Self {
+        Self
+    }
+}
+
+impl From<LfuContext> for CacheContext {
+    fn from(_: LfuContext) -> Self {
+        CacheContext::Default
+    }
+}
 
 #[derive(Debug, PartialEq, Eq)]
 enum Queue {
@@ -473,7 +486,7 @@ mod tests {
             let ptrs = (0..100)
                 .map(|i| {
                     let mut handle = Box::new(TestLfuHandle::new());
-                    handle.init(i, i, i, 1, ());
+                    handle.init(i, i, i, 1, LfuContext);
                     NonNull::new_unchecked(Box::into_raw(handle))
                 })
                 .collect_vec();
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index e036bd19..bf8c36cb 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -23,7 +23,7 @@ use foyer_intrusive::{
 use crate::{
     eviction::Eviction,
     handle::{BaseHandle, Handle},
-    Key, Value,
+    CacheContext, Key, Value,
 };
 
 #[derive(Debug)]
@@ -45,9 +45,21 @@ pub enum LruContext {
     LowPriority,
 }
 
-impl Default for LruContext {
-    fn default() -> Self {
-        Self::HighPriority
+impl From<CacheContext> for LruContext {
+    fn from(value: CacheContext) -> Self {
+        match value {
+            CacheContext::Default => Self::HighPriority,
+            CacheContext::LruPriorityLow => Self::LowPriority,
+        }
+    }
+}
+
+impl From<LruContext> for CacheContext {
+    fn from(value: LruContext) -> Self {
+        match value {
+            LruContext::HighPriority => CacheContext::Default,
+            LruContext::LowPriority => CacheContext::LruPriorityLow,
+        }
     }
 }
 
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
index f1b2d0f4..46f33176 100644
--- a/foyer-memory/src/handle.rs
+++ b/foyer-memory/src/handle.rs
@@ -14,7 +14,7 @@
 
 use bitflags::bitflags;
 
-use crate::{Context, Key, Value};
+use crate::{context::Context, Key, Value};
 
 bitflags! {
     #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index c3a4ad92..0b026735 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -69,13 +69,12 @@
 
 pub trait Key: Send + Sync + 'static + std::hash::Hash + Eq + Ord {}
 pub trait Value: Send + Sync + 'static {}
-pub trait Context: Send + Sync + 'static + Default {}
 
 impl<T: Send + Sync + 'static + std::hash::Hash + Eq + Ord> Key for T {}
 impl<T: Send + Sync + 'static> Value for T {}
-impl<T: Send + Sync + 'static + Default> Context for T {}
 
 mod cache;
+mod context;
 mod eviction;
 mod handle;
 mod indexer;
diff --git a/foyer-memory/src/listener.rs b/foyer-memory/src/listener.rs
index ec088e88..a858c306 100644
--- a/foyer-memory/src/listener.rs
+++ b/foyer-memory/src/listener.rs
@@ -14,42 +14,38 @@
 
 use std::marker::PhantomData;
 
-use crate::{Context, Key, Value};
+use crate::{CacheContext, Key, Value};
 
-pub trait CacheEventListener<K, V, C>: Send + Sync + 'static
+pub trait CacheEventListener<K, V>: Send + Sync + 'static
 where
     K: Key,
     V: Value,
-    C: Context,
 {
     /// The function is called when an entry is released by the cache and all external users.
     ///
     /// The arguments includes the key and value with ownership.
-    fn on_release(&self, key: K, value: V, context: C, charges: usize);
+    fn on_release(&self, key: K, value: V, context: CacheContext, charges: usize);
 }
 
-pub struct DefaultCacheEventListener<K, V, C>(PhantomData<(K, V, C)>)
+pub struct DefaultCacheEventListener<K, V>(PhantomData<(K, V)>)
 where
     K: Key,
-    V: Value,
-    C: Context;
+    V: Value;
 
-impl<K, V, C> Default for DefaultCacheEventListener<K, V, C>
+impl<K, V> Default for DefaultCacheEventListener<K, V>
 where
     K: Key,
     V: Value,
-    C: Context,
 {
     fn default() -> Self {
         Self(Default::default())
     }
 }
 
-impl<K, V, C> CacheEventListener<K, V, C> for DefaultCacheEventListener<K, V, C>
+impl<K, V> CacheEventListener<K, V> for DefaultCacheEventListener<K, V>
 where
     K: Key,
     V: Value,
-    C: Context,
 {
-    fn on_release(&self, _key: K, _value: V, _context: C, _charges: usize) {}
+    fn on_release(&self, _key: K, _value: V, _context: CacheContext, _charges: usize) {}
 }
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index a99f0c26..6795618a 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -25,29 +25,27 @@ use crate::{
     listener::DefaultCacheEventListener,
 };
 
-pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     Cache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
-pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
-    CacheConfig<Fifo<K, V>, L, S>;
-pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V, FifoContext>, S = RandomState> =
+pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Fifo<K, V>, L, S>;
+pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     CacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
 
-pub type LruCache<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+pub type LruCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     Cache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
-pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
-    CacheConfig<Lru<K, V>, L, S>;
-pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LruContext>, S = RandomState> =
+pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lru<K, V>, L, S>;
+pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
 
-pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     Cache<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
-pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
-    CacheConfig<Lfu<K, V>, L, S>;
-pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V, LfuContext>, S = RandomState> =
+pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lfu<K, V>, L, S>;
+pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     CacheEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
 
 pub use crate::{
     cache::Entry,
+    context::CacheContext,
     eviction::{
         fifo::{FifoConfig, FifoContext},
         lfu::{LfuConfig, LfuContext},

From 7c3971fc1964061f240e749e523fa0ac8271664f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 12 Mar 2024 12:04:46 +0800
Subject: [PATCH 227/261] refactor: unify in-memory cache to simplify usage
 (#288)

* refactor: unify in-memory cache to simplify usage

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: export more necessary structs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: make cache context clone + copy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: add interface to chekc entry type

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: export entry state with enum

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix trait bound

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: add default generic type for entry

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: export entry state

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: add eq for cache context

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix unimplemented path

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs         | 1013 +++++++----------------------
 foyer-memory/src/context.rs       |    5 +-
 foyer-memory/src/eviction/fifo.rs |    2 +-
 foyer-memory/src/eviction/lfu.rs  |    3 +-
 foyer-memory/src/eviction/lru.rs  |    2 +-
 foyer-memory/src/generic.rs       |  949 +++++++++++++++++++++++++++
 foyer-memory/src/lib.rs           |    1 +
 foyer-memory/src/prelude.rs       |   40 +-
 8 files changed, 1202 insertions(+), 813 deletions(-)
 create mode 100644 foyer-memory/src/generic.rs

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index d560bb0c..1c902b18 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -12,944 +12,417 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    future::Future,
-    hash::BuildHasher,
-    ops::Deref,
-    ptr::NonNull,
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc,
-    },
-};
+use std::{hash::BuildHasher, ops::Deref, sync::Arc};
 
 use ahash::RandomState;
-use crossbeam::queue::ArrayQueue;
-use futures::FutureExt;
-use hashbrown::hash_map::{Entry as HashMapEntry, HashMap};
-use itertools::Itertools;
-use parking_lot::Mutex;
-use tokio::{sync::oneshot, task::JoinHandle};
+use futures::{Future, FutureExt};
+use tokio::sync::oneshot;
 
 use crate::{
-    eviction::Eviction, handle::Handle, indexer::Indexer, listener::CacheEventListener, metrics::Metrics, CacheContext,
+    context::CacheContext,
+    eviction::{
+        fifo::{Fifo, FifoHandle},
+        lfu::{Lfu, LfuHandle},
+        lru::{Lru, LruHandle},
+    },
+    generic::{CacheConfig, GenericCache, GenericCacheEntry, GenericEntry},
+    indexer::HashTableIndexer,
+    listener::{CacheEventListener, DefaultCacheEventListener},
+    metrics::Metrics,
     Key, Value,
 };
 
-struct CacheSharedState<T, L> {
-    metrics: Metrics,
-    /// The object pool to avoid frequent handle allocating, shared by all shards.
-    object_pool: ArrayQueue<Box<T>>,
-    listener: L,
-}
-
-#[expect(clippy::type_complexity)]
-struct CacheShard<K, V, H, E, I, L, S>
+pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Fifo<K, V>, L, S>;
+pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+pub type FifoEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S, ER>;
+
+pub type LruCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lru<K, V>, L, S>;
+pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+pub type LruEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S, ER>;
+
+pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCache<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lfu<K, V>, L, S>;
+pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCacheEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+pub type LfuEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S, ER>;
+
+pub enum CacheEntry<K, V, L, S = RandomState>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    indexer: I,
-    eviction: E,
-
-    capacity: usize,
-    usage: Arc<AtomicUsize>,
-
-    waiters: HashMap<K, Vec<oneshot::Sender<CacheEntry<K, V, H, E, I, L, S>>>>,
-
-    state: Arc<CacheSharedState<H, L>>,
+    Fifo(FifoCacheEntry<K, V, L, S>),
+    Lru(LruCacheEntry<K, V, L, S>),
+    Lfu(LfuCacheEntry<K, V, L, S>),
 }
 
-impl<K, V, H, E, I, L, S> CacheShard<K, V, H, E, I, L, S>
+impl<K, V, L, S> Clone for CacheEntry<K, V, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    fn new(
-        capacity: usize,
-        eviction_config: &E::Config,
-        usage: Arc<AtomicUsize>,
-        context: Arc<CacheSharedState<H, L>>,
-    ) -> Self {
-        let indexer = I::new();
-        let eviction = unsafe { E::new(capacity, eviction_config) };
-        let waiters = HashMap::default();
-        Self {
-            indexer,
-            eviction,
-            capacity,
-            usage,
-            waiters,
-            state: context,
-        }
-    }
-
-    /// Insert a new entry into the cache. The handle for the new entry is returned.
-    unsafe fn insert(
-        &mut self,
-        hash: u64,
-        key: K,
-        value: V,
-        charge: usize,
-        context: H::Context,
-        last_reference_entries: &mut Vec<(K, V, H::Context, usize)>,
-    ) -> NonNull<H> {
-        let mut handle = self.state.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
-        handle.init(hash, key, value, charge, context);
-        let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
-
-        self.evict(charge, last_reference_entries);
-
-        debug_assert!(!ptr.as_ref().base().is_in_indexer());
-        if let Some(old) = self.indexer.insert(ptr) {
-            self.state.metrics.replace.fetch_add(1, Ordering::Relaxed);
-
-            debug_assert!(!old.as_ref().base().is_in_indexer());
-            if old.as_ref().base().is_in_eviction() {
-                self.eviction.remove(old);
-            }
-            debug_assert!(!old.as_ref().base().is_in_eviction());
-            // Because the `old` handle is removed from the indexer, it will not be reinserted again.
-            if let Some(entry) = self.try_release_handle(old, false) {
-                last_reference_entries.push(entry);
-            }
-        } else {
-            self.state.metrics.insert.fetch_add(1, Ordering::Relaxed);
-        }
-        self.eviction.push(ptr);
-
-        debug_assert!(ptr.as_ref().base().is_in_indexer());
-        debug_assert!(ptr.as_ref().base().is_in_indexer());
-
-        self.usage.fetch_add(charge, Ordering::Relaxed);
-        ptr.as_mut().base_mut().inc_refs();
-
-        ptr
-    }
-
-    unsafe fn get(&mut self, hash: u64, key: &K) -> Option<NonNull<H>> {
-        let mut ptr = match self.indexer.get(hash, key) {
-            Some(ptr) => {
-                self.state.metrics.hit.fetch_add(1, Ordering::Relaxed);
-                ptr
-            }
-            None => {
-                self.state.metrics.miss.fetch_add(1, Ordering::Relaxed);
-                return None;
-            }
-        };
-        let base = ptr.as_mut().base_mut();
-        debug_assert!(base.is_in_indexer());
-
-        base.inc_refs();
-        self.eviction.access(ptr);
-
-        Some(ptr)
-    }
-
-    /// Remove a key from the cache.
-    ///
-    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V, H::Context, usize)> {
-        let ptr = self.indexer.remove(hash, key)?;
-        self.state.metrics.remove.fetch_add(1, Ordering::Relaxed);
-        if ptr.as_ref().base().is_in_eviction() {
-            self.eviction.remove(ptr);
-        }
-        debug_assert!(!ptr.as_ref().base().is_in_indexer());
-        debug_assert!(!ptr.as_ref().base().is_in_eviction());
-        self.try_release_handle(ptr, false)
-    }
-
-    /// Clear all cache entries.
-    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
-        // TODO(MrCroxx): Avoid collecting here?
-        let ptrs = self.indexer.drain().collect_vec();
-        let eptrs = self.eviction.clear();
-
-        // Assert that the handles in the indexer covers the handles in the eviction container.
-        if cfg!(debug_assertions) {
-            use std::{collections::HashSet as StdHashSet, hash::RandomState as StdRandomState};
-            let ptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(ptrs.iter().copied());
-            let eptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(eptrs.iter().copied());
-            debug_assert!((&eptrs - &ptrs).is_empty());
-        }
-
-        self.state.metrics.remove.fetch_add(ptrs.len(), Ordering::Relaxed);
-
-        // The handles in the indexer covers the handles in the eviction container.
-        // So only the handles drained from the indexer need to be released.
-        for ptr in ptrs {
-            debug_assert!(!ptr.as_ref().base().is_in_indexer());
-            if let Some(entry) = self.try_release_handle(ptr, false) {
-                last_reference_entries.push(entry);
-            }
-        }
-    }
-
-    unsafe fn evict(&mut self, charge: usize, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
-        while self.usage.load(Ordering::Relaxed) + charge > self.capacity
-            && let Some(evicted) = self.eviction.pop()
-        {
-            self.state.metrics.evict.fetch_add(1, Ordering::Relaxed);
-            let base = evicted.as_ref().base();
-            debug_assert!(base.is_in_indexer());
-            debug_assert!(!base.is_in_eviction());
-            if let Some(entry) = self.try_release_handle(evicted, false) {
-                last_reference_entries.push(entry);
-            }
-        }
-    }
-
-    /// Release a handle used by an external user.
-    ///
-    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn try_release_external_handle(&mut self, mut ptr: NonNull<H>) -> Option<(K, V, H::Context, usize)> {
-        ptr.as_mut().base_mut().dec_refs();
-        self.try_release_handle(ptr, true)
-    }
-
-    /// Try release handle if there is no external reference and no reinsertion is needed.
-    ///
-    /// Return the entry if the handle is released.
-    ///
-    /// Recycle it if possible.
-    unsafe fn try_release_handle(&mut self, mut ptr: NonNull<H>, reinsert: bool) -> Option<(K, V, H::Context, usize)> {
-        let base = ptr.as_mut().base_mut();
-
-        if base.has_refs() {
-            return None;
-        }
-
-        debug_assert!(base.is_inited());
-        debug_assert!(!base.has_refs());
-
-        // If the entry is not updated or removed from the cache, try to reinsert it or remove it from the indexer and
-        // the eviction container.
-        if base.is_in_indexer() {
-            // The usage is higher than the capacity means most handles are held externally,
-            // the cache shard cannot release enough charges for the new inserted entries.
-            // In this case, the reinsertion should be given up.
-            if reinsert && self.usage.load(Ordering::Relaxed) <= self.capacity {
-                let was_in_eviction = base.is_in_eviction();
-                self.eviction.reinsert(ptr);
-                if ptr.as_ref().base().is_in_eviction() {
-                    if was_in_eviction {
-                        self.state.metrics.reinsert.fetch_add(1, Ordering::Relaxed);
-                    }
-                    return None;
-                }
-            }
-
-            // If the entry has not been reinserted, remove it from the indexer and the eviction container (if needed).
-            self.indexer.remove(base.hash(), base.key());
-            if ptr.as_ref().base().is_in_eviction() {
-                self.eviction.remove(ptr);
-            }
+    fn clone(&self) -> Self {
+        match self {
+            Self::Fifo(entry) => Self::Fifo(entry.clone()),
+            Self::Lru(entry) => Self::Lru(entry.clone()),
+            Self::Lfu(entry) => Self::Lfu(entry.clone()),
         }
-
-        // Here the handle is neither in the indexer nor in the eviction container.
-        debug_assert!(!base.is_in_indexer());
-        debug_assert!(!base.is_in_eviction());
-        debug_assert!(!base.has_refs());
-
-        self.state.metrics.release.fetch_add(1, Ordering::Relaxed);
-
-        self.usage.fetch_sub(base.charge(), Ordering::Relaxed);
-        let entry = base.take();
-
-        let handle = Box::from_raw(ptr.as_ptr());
-        let _ = self.state.object_pool.push(handle);
-
-        Some(entry)
     }
 }
 
-impl<K, V, H, E, I, L, S> Drop for CacheShard<K, V, H, E, I, L, S>
+impl<K, V, L, S> Deref for CacheEntry<K, V, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    fn drop(&mut self) {
-        unsafe { self.clear(&mut vec![]) }
+    type Target = V;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            CacheEntry::Fifo(entry) => entry.deref(),
+            CacheEntry::Lru(entry) => entry.deref(),
+            CacheEntry::Lfu(entry) => entry.deref(),
+        }
     }
 }
 
-pub struct CacheConfig<E, L, S = RandomState>
+impl<K, V, L, S> From<FifoCacheEntry<K, V, L, S>> for CacheEntry<K, V, L, S>
 where
-    E: Eviction,
-    L: CacheEventListener<<E::Handle as Handle>::Key, <E::Handle as Handle>::Value>,
+    K: Key,
+    V: Value,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub capacity: usize,
-    pub shards: usize,
-    pub eviction_config: E::Config,
-    pub object_pool_capacity: usize,
-    pub hash_builder: S,
-    pub event_listener: L,
+    fn from(entry: FifoCacheEntry<K, V, L, S>) -> Self {
+        Self::Fifo(entry)
+    }
 }
 
-#[expect(clippy::type_complexity)]
-pub enum Entry<K, V, H, E, I, L, S, ER>
+impl<K, V, L, S> From<LruCacheEntry<K, V, L, S>> for CacheEntry<K, V, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
-    ER: std::error::Error,
 {
-    Invalid,
-    Hit(CacheEntry<K, V, H, E, I, L, S>),
-    Wait(oneshot::Receiver<CacheEntry<K, V, H, E, I, L, S>>),
-    Miss(JoinHandle<std::result::Result<CacheEntry<K, V, H, E, I, L, S>, ER>>),
+    fn from(entry: LruCacheEntry<K, V, L, S>) -> Self {
+        Self::Lru(entry)
+    }
 }
 
-impl<K, V, H, E, I, L, S, ER> Default for Entry<K, V, H, E, I, L, S, ER>
+impl<K, V, L, S> From<LfuCacheEntry<K, V, L, S>> for CacheEntry<K, V, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
-    ER: std::error::Error,
 {
-    fn default() -> Self {
-        Self::Invalid
+    fn from(entry: LfuCacheEntry<K, V, L, S>) -> Self {
+        Self::Lfu(entry)
     }
 }
 
-impl<K, V, H, E, I, L, S, ER> Future for Entry<K, V, H, E, I, L, S, ER>
+impl<K, V, L, S> CacheEntry<K, V, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
-    ER: std::error::Error + From<oneshot::error::RecvError>,
 {
-    type Output = std::result::Result<CacheEntry<K, V, H, E, I, L, S>, ER>;
+    pub fn key(&self) -> &K {
+        match self {
+            CacheEntry::Fifo(entry) => entry.key(),
+            CacheEntry::Lru(entry) => entry.key(),
+            CacheEntry::Lfu(entry) => entry.key(),
+        }
+    }
 
-    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
-        match &mut *self {
-            Self::Invalid => unreachable!(),
-            Self::Hit(_) => std::task::Poll::Ready(Ok(match std::mem::take(&mut *self) {
-                Entry::Hit(entry) => entry,
-                _ => unreachable!(),
-            })),
-            Self::Wait(waiter) => waiter.poll_unpin(cx).map_err(|err| err.into()),
-            Self::Miss(join_handle) => join_handle.poll_unpin(cx).map(|join_result| join_result.unwrap()),
+    pub fn value(&self) -> &V {
+        match self {
+            CacheEntry::Fifo(entry) => entry.value(),
+            CacheEntry::Lru(entry) => entry.value(),
+            CacheEntry::Lfu(entry) => entry.value(),
+        }
+    }
+
+    pub fn context(&self) -> CacheContext {
+        match self {
+            CacheEntry::Fifo(entry) => entry.context().clone().into(),
+            CacheEntry::Lru(entry) => entry.context().clone().into(),
+            CacheEntry::Lfu(entry) => entry.context().clone().into(),
+        }
+    }
+
+    pub fn charge(&self) -> usize {
+        match self {
+            CacheEntry::Fifo(entry) => entry.charge(),
+            CacheEntry::Lru(entry) => entry.charge(),
+            CacheEntry::Lfu(entry) => entry.charge(),
+        }
+    }
+
+    pub fn refs(&self) -> usize {
+        match self {
+            CacheEntry::Fifo(entry) => entry.refs(),
+            CacheEntry::Lru(entry) => entry.refs(),
+            CacheEntry::Lfu(entry) => entry.refs(),
         }
     }
 }
 
-#[expect(clippy::type_complexity)]
-pub struct Cache<K, V, H, E, I, L, S = RandomState>
+pub enum Cache<K, V, L, S = RandomState>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    shards: Vec<Mutex<CacheShard<K, V, H, E, I, L, S>>>,
-
-    capacity: usize,
-    usages: Vec<Arc<AtomicUsize>>,
-
-    context: Arc<CacheSharedState<H, L>>,
-
-    hash_builder: S,
+    Fifo(Arc<FifoCache<K, V, L, S>>),
+    Lru(Arc<LruCache<K, V, L, S>>),
+    Lfu(Arc<LfuCache<K, V, L, S>>),
 }
 
-impl<K, V, H, E, I, L, S> Cache<K, V, H, E, I, L, S>
+impl<K, V, L, S> Clone for Cache<K, V, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn new(config: CacheConfig<E, L, S>) -> Self {
-        let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
-        let context = Arc::new(CacheSharedState {
-            metrics: Metrics::default(),
-            object_pool: ArrayQueue::new(config.object_pool_capacity),
-            listener: config.event_listener,
-        });
-
-        let shard_capacity = config.capacity / config.shards;
-
-        let shards = usages
-            .iter()
-            .map(|usage| CacheShard::new(shard_capacity, &config.eviction_config, usage.clone(), context.clone()))
-            .map(Mutex::new)
-            .collect_vec();
-
-        Self {
-            shards,
-            capacity: config.capacity,
-            usages,
-            context,
-            hash_builder: config.hash_builder,
+    fn clone(&self) -> Self {
+        match self {
+            Self::Fifo(cache) => Self::Fifo(cache.clone()),
+            Self::Lru(cache) => Self::Lru(cache.clone()),
+            Self::Lfu(cache) => Self::Lfu(cache.clone()),
         }
     }
+}
+
+impl<K, V, L, S> Cache<K, V, L, S>
+where
+    K: Key,
+    V: Value,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn fifo(config: FifoCacheConfig<K, V, L, S>) -> Self {
+        Self::Fifo(Arc::new(GenericCache::new(config)))
+    }
+
+    pub fn lru(config: LruCacheConfig<K, V, L, S>) -> Self {
+        Self::Lru(Arc::new(GenericCache::new(config)))
+    }
 
-    pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> CacheEntry<K, V, H, E, I, L, S> {
-        self.insert_with_context(key, value, charge, CacheContext::default())
+    pub fn lfu(config: LfuCacheConfig<K, V, L, S>) -> Self {
+        Self::Lfu(Arc::new(GenericCache::new(config)))
+    }
+
+    pub fn insert(&self, key: K, value: V, charge: usize) -> CacheEntry<K, V, L, S> {
+        match self {
+            Cache::Fifo(cache) => cache.insert(key, value, charge).into(),
+            Cache::Lru(cache) => cache.insert(key, value, charge).into(),
+            Cache::Lfu(cache) => cache.insert(key, value, charge).into(),
+        }
     }
 
     pub fn insert_with_context(
-        self: &Arc<Self>,
+        &self,
         key: K,
         value: V,
         charge: usize,
         context: CacheContext,
-    ) -> CacheEntry<K, V, H, E, I, L, S> {
-        let hash = self.hash_builder.hash_one(&key);
-
-        let mut to_deallocate = vec![];
-
-        let (entry, waiters) = unsafe {
-            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
-            let waiters = shard.waiters.remove(&key);
-            let mut ptr = shard.insert(hash, key, value, charge, context.into(), &mut to_deallocate);
-            if let Some(waiters) = waiters.as_ref() {
-                ptr.as_mut().base_mut().inc_refs_by(waiters.len());
-            }
-            let entry = CacheEntry {
-                cache: self.clone(),
-                ptr,
-            };
-            (entry, waiters)
-        };
-
-        if let Some(waiters) = waiters {
-            for waiter in waiters {
-                let _ = waiter.send(CacheEntry {
-                    cache: self.clone(),
-                    ptr: entry.ptr,
-                });
-            }
+    ) -> CacheEntry<K, V, L, S> {
+        match self {
+            Cache::Fifo(cache) => cache.insert_with_context(key, value, charge, context).into(),
+            Cache::Lru(cache) => cache.insert_with_context(key, value, charge, context).into(),
+            Cache::Lfu(cache) => cache.insert_with_context(key, value, charge, context).into(),
         }
-
-        // Do not deallocate data within the lock section.
-        for (key, value, context, charges) in to_deallocate {
-            self.context.listener.on_release(key, value, context.into(), charges)
-        }
-
-        entry
     }
 
     pub fn remove(&self, key: &K) {
-        let hash = self.hash_builder.hash_one(key);
-
-        let entry = unsafe {
-            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
-            shard.remove(hash, key)
-        };
-
-        // Do not deallocate data within the lock section.
-        if let Some((key, value, context, charges)) = entry {
-            self.context.listener.on_release(key, value, context.into(), charges);
+        match self {
+            Cache::Fifo(cache) => cache.remove(key),
+            Cache::Lru(cache) => cache.remove(key),
+            Cache::Lfu(cache) => cache.remove(key),
         }
     }
 
-    pub fn get(self: &Arc<Self>, key: &K) -> Option<CacheEntry<K, V, H, E, I, L, S>> {
-        let hash = self.hash_builder.hash_one(key);
-
-        unsafe {
-            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
-            shard.get(hash, key).map(|ptr| CacheEntry {
-                cache: self.clone(),
-                ptr,
-            })
+    pub fn get(&self, key: &K) -> Option<CacheEntry<K, V, L, S>> {
+        match self {
+            Cache::Fifo(cache) => cache.get(key).map(CacheEntry::from),
+            Cache::Lru(cache) => cache.get(key).map(CacheEntry::from),
+            Cache::Lfu(cache) => cache.get(key).map(CacheEntry::from),
         }
     }
 
     pub fn clear(&self) {
-        let mut to_deallocate = vec![];
-        for shard in self.shards.iter() {
-            let mut shard = shard.lock();
-            unsafe { shard.clear(&mut to_deallocate) };
+        match self {
+            Cache::Fifo(cache) => cache.clear(),
+            Cache::Lru(cache) => cache.clear(),
+            Cache::Lfu(cache) => cache.clear(),
         }
     }
 
     pub fn capacity(&self) -> usize {
-        self.capacity
+        match self {
+            Cache::Fifo(cache) => cache.capacity(),
+            Cache::Lru(cache) => cache.capacity(),
+            Cache::Lfu(cache) => cache.capacity(),
+        }
     }
 
     pub fn usage(&self) -> usize {
-        self.usages.iter().map(|usage| usage.load(Ordering::Relaxed)).sum()
+        match self {
+            Cache::Fifo(cache) => cache.usage(),
+            Cache::Lru(cache) => cache.usage(),
+            Cache::Lfu(cache) => cache.usage(),
+        }
     }
 
     pub fn metrics(&self) -> &Metrics {
-        &self.context.metrics
-    }
-
-    unsafe fn try_release_external_handle(&self, ptr: NonNull<H>) {
-        let entry = {
-            let base = ptr.as_ref().base();
-            let mut shard = self.shards[base.hash() as usize % self.shards.len()].lock();
-            shard.try_release_external_handle(ptr)
-        };
-
-        // Do not deallocate data within the lock section.
-        if let Some((key, value, context, charges)) = entry {
-            self.context.listener.on_release(key, value, context.into(), charges);
+        match self {
+            Cache::Fifo(cache) => cache.metrics(),
+            Cache::Lru(cache) => cache.metrics(),
+            Cache::Lfu(cache) => cache.metrics(),
         }
     }
 }
 
-// TODO(MrCroxx): use `hashbrown::HashTable` with `Handle` may relax the `Clone` bound?
-impl<K, V, H, E, I, L, S> Cache<K, V, H, E, I, L, S>
+pub enum Entry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState>
 where
     K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
-    L: CacheEventListener<K, V>,
-    S: BuildHasher + Send + Sync + 'static,
-{
-    pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> Entry<K, V, H, E, I, L, S, ER>
-    where
-        F: FnOnce() -> FU,
-        FU: Future<Output = std::result::Result<(V, usize, Option<H::Context>), ER>> + Send + 'static,
-        ER: std::error::Error + Send + 'static,
-    {
-        let hash = self.hash_builder.hash_one(&key);
-
-        unsafe {
-            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
-            if let Some(ptr) = shard.get(hash, &key) {
-                return Entry::Hit(CacheEntry {
-                    cache: self.clone(),
-                    ptr,
-                });
-            }
-            let entry = match shard.waiters.entry(key.clone()) {
-                HashMapEntry::Occupied(mut o) => {
-                    let (tx, rx) = oneshot::channel();
-                    o.get_mut().push(tx);
-                    Entry::Wait(rx)
-                }
-                HashMapEntry::Vacant(v) => {
-                    v.insert(vec![]);
-                    let cache = self.clone();
-                    let future = f();
-                    let join = tokio::spawn(async move {
-                        let (value, charge, context) = match future.await {
-                            Ok((value, charge, context)) => (value, charge, context),
-                            Err(e) => {
-                                let mut shard = cache.shards[hash as usize % cache.shards.len()].lock();
-                                shard.waiters.remove(&key);
-                                return Err(e);
-                            }
-                        };
-
-                        let entry = if let Some(context) = context {
-                            cache.insert_with_context(key, value, charge, context.into())
-                        } else {
-                            cache.insert(key, value, charge)
-                        };
-
-                        Ok(entry)
-                    });
-                    Entry::Miss(join)
-                }
-            };
-            match entry {
-                Entry::Wait(_) => shard.state.metrics.queue.fetch_add(1, Ordering::Relaxed),
-                Entry::Miss(_) => shard.state.metrics.fetch.fetch_add(1, Ordering::Relaxed),
-                _ => unreachable!(),
-            };
-            entry
-        }
-    }
-}
-
-pub struct CacheEntry<K, V, H, E, I, L, S = RandomState>
-where
-    K: Key,
-    V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    ER: std::error::Error,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    cache: Arc<Cache<K, V, H, E, I, L, S>>,
-    ptr: NonNull<H>,
+    Fifo(FifoEntry<K, V, ER, L, S>),
+    Lru(LruEntry<K, V, ER, L, S>),
+    Lfu(LfuEntry<K, V, ER, L, S>),
 }
 
-impl<K, V, H, E, I, L, S> CacheEntry<K, V, H, E, I, L, S>
+impl<K, V, ER, L, S> From<FifoEntry<K, V, ER, L, S>> for Entry<K, V, ER, L, S>
 where
-    K: Key,
+    K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    ER: std::error::Error,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn key(&self) -> &H::Key {
-        unsafe { self.ptr.as_ref().base().key() }
-    }
-
-    pub fn value(&self) -> &H::Value {
-        unsafe { self.ptr.as_ref().base().value() }
-    }
-
-    pub fn context(&self) -> &H::Context {
-        unsafe { self.ptr.as_ref().base().context() }
-    }
-
-    pub fn charge(&self) -> usize {
-        unsafe { self.ptr.as_ref().base().charge() }
-    }
-
-    pub fn refs(&self) -> usize {
-        unsafe { self.ptr.as_ref().base().refs() }
+    fn from(entry: FifoEntry<K, V, ER, L, S>) -> Self {
+        Self::Fifo(entry)
     }
 }
 
-impl<K, V, H, E, I, L, S> Clone for CacheEntry<K, V, H, E, I, L, S>
+impl<K, V, ER, L, S> From<LruEntry<K, V, ER, L, S>> for Entry<K, V, ER, L, S>
 where
-    K: Key,
+    K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    ER: std::error::Error,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    fn clone(&self) -> Self {
-        let mut ptr = self.ptr;
-
-        unsafe {
-            let base = ptr.as_mut().base_mut();
-            debug_assert!(base.has_refs());
-            base.inc_refs();
-        }
-
-        Self {
-            cache: self.cache.clone(),
-            ptr,
-        }
+    fn from(entry: LruEntry<K, V, ER, L, S>) -> Self {
+        Self::Lru(entry)
     }
 }
 
-impl<K, V, H, E, I, L, S> Drop for CacheEntry<K, V, H, E, I, L, S>
+impl<K, V, ER, L, S> From<LfuEntry<K, V, ER, L, S>> for Entry<K, V, ER, L, S>
 where
-    K: Key,
+    K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    ER: std::error::Error,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    fn drop(&mut self) {
-        unsafe { self.cache.try_release_external_handle(self.ptr) }
+    fn from(entry: LfuEntry<K, V, ER, L, S>) -> Self {
+        Self::Lfu(entry)
     }
 }
 
-impl<K, V, H, E, I, L, S> Deref for CacheEntry<K, V, H, E, I, L, S>
+impl<K, V, ER, L, S> Future for Entry<K, V, ER, L, S>
 where
-    K: Key,
+    K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    ER: std::error::Error + From<oneshot::error::RecvError>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    type Target = V;
+    type Output = std::result::Result<CacheEntry<K, V, L, S>, ER>;
 
-    fn deref(&self) -> &Self::Target {
-        self.value()
+    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
+        match &mut *self {
+            Entry::Fifo(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)),
+            Entry::Lru(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)),
+            Entry::Lfu(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)),
+        }
     }
 }
 
-unsafe impl<K, V, H, E, I, L, S> Send for CacheEntry<K, V, H, E, I, L, S>
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EntryState {
+    Hit,
+    Wait,
+    Miss,
+}
+
+impl<K, V, ER, L, S> Entry<K, V, ER, L, S>
 where
-    K: Key,
+    K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    ER: std::error::Error,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
+    pub fn state(&self) -> EntryState {
+        match self {
+            Entry::Fifo(FifoEntry::Hit(_)) | Entry::Lru(LruEntry::Hit(_)) | Entry::Lfu(LfuEntry::Hit(_)) => {
+                EntryState::Hit
+            }
+            Entry::Fifo(FifoEntry::Wait(_)) | Entry::Lru(LruEntry::Wait(_)) | Entry::Lfu(LfuEntry::Wait(_)) => {
+                EntryState::Wait
+            }
+            Entry::Fifo(FifoEntry::Miss(_)) | Entry::Lru(LruEntry::Miss(_)) | Entry::Lfu(LfuEntry::Miss(_)) => {
+                EntryState::Miss
+            }
+            _ => unreachable!(),
+        }
+    }
 }
-unsafe impl<K, V, H, E, I, L, S> Sync for CacheEntry<K, V, H, E, I, L, S>
+
+impl<K, V, L, S> Cache<K, V, L, S>
 where
-    K: Key,
+    K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-}
-
-#[cfg(test)]
-mod tests {
-    use rand::{rngs::SmallRng, RngCore, SeedableRng};
-
-    use super::*;
-    use crate::{
-        eviction::{
-            fifo::{FifoConfig, FifoHandle},
-            lru::LruConfig,
-            test_utils::TestEviction,
-        },
-        listener::DefaultCacheEventListener,
-        prelude::{FifoCache, FifoCacheConfig, FifoCacheEntry, LruCache, LruCacheConfig, LruCacheEntry},
-    };
-
-    fn is_send_sync_static<T: Send + Sync + 'static>() {}
-
-    #[test]
-    fn test_send_sync_static() {
-        is_send_sync_static::<FifoCache<(), ()>>();
-        is_send_sync_static::<FifoCacheConfig<(), ()>>();
-        is_send_sync_static::<LruCache<(), ()>>();
-        is_send_sync_static::<LruCacheConfig<(), ()>>();
-    }
-
-    #[test]
-    fn test_cache_fuzzy() {
-        const CAPACITY: usize = 256;
-
-        let config = FifoCacheConfig {
-            capacity: CAPACITY,
-            shards: 4,
-            eviction_config: FifoConfig {},
-            object_pool_capacity: 16,
-            hash_builder: RandomState::default(),
-            event_listener: DefaultCacheEventListener::default(),
-        };
-        let cache = Arc::new(FifoCache::<u64, u64>::new(config));
-
-        let mut rng = SmallRng::seed_from_u64(114514);
-        for _ in 0..100000 {
-            let key = rng.next_u64();
-            if let Some(entry) = cache.get(&key) {
-                assert_eq!(key, *entry);
-                drop(entry);
-                continue;
-            }
-            cache.insert(key, key, 1);
+    pub fn entry<F, FU, ER>(&self, key: K, f: F) -> Entry<K, V, ER, L, S>
+    where
+        F: FnOnce() -> FU,
+        FU: Future<Output = std::result::Result<(V, usize, CacheContext), ER>> + Send + 'static,
+        ER: std::error::Error + Send + 'static,
+    {
+        match self {
+            Cache::Fifo(cache) => Entry::from(cache.entry(key, f)),
+            Cache::Lru(cache) => Entry::from(cache.entry(key, f)),
+            Cache::Lfu(cache) => Entry::from(cache.entry(key, f)),
         }
-        assert_eq!(cache.usage(), CAPACITY);
-    }
-
-    fn fifo(capacity: usize) -> Arc<FifoCache<u64, String>> {
-        let config = FifoCacheConfig {
-            capacity,
-            shards: 1,
-            eviction_config: FifoConfig {},
-            object_pool_capacity: 1,
-            hash_builder: RandomState::default(),
-            event_listener: DefaultCacheEventListener::default(),
-        };
-        Arc::new(FifoCache::<u64, String>::new(config))
-    }
-
-    fn lru(capacity: usize) -> Arc<LruCache<u64, String>> {
-        let config = LruCacheConfig {
-            capacity,
-            shards: 1,
-            eviction_config: LruConfig {
-                high_priority_pool_ratio: 0.0,
-            },
-            object_pool_capacity: 1,
-            hash_builder: RandomState::default(),
-            event_listener: DefaultCacheEventListener::default(),
-        };
-        Arc::new(LruCache::<u64, String>::new(config))
-    }
-
-    fn insert_fifo(cache: &Arc<FifoCache<u64, String>>, key: u64, value: &str) -> FifoCacheEntry<u64, String> {
-        cache.insert(key, value.to_string(), value.len())
-    }
-
-    fn insert_lru(cache: &Arc<LruCache<u64, String>>, key: u64, value: &str) -> LruCacheEntry<u64, String> {
-        cache.insert(key, value.to_string(), value.len())
-    }
-
-    #[test]
-    fn test_reference_count() {
-        let cache = fifo(100);
-
-        let refs = |ptr: NonNull<FifoHandle<u64, String>>| unsafe { ptr.as_ref().base().refs() };
-
-        let e1 = insert_fifo(&cache, 42, "the answer to life, the universe, and everything");
-        let ptr = e1.ptr;
-        assert_eq!(refs(ptr), 1);
-
-        let e2 = cache.get(&42).unwrap();
-        assert_eq!(refs(ptr), 2);
-
-        let e3 = e2.clone();
-        assert_eq!(refs(ptr), 3);
-
-        drop(e2);
-        assert_eq!(refs(ptr), 2);
-
-        drop(e3);
-        assert_eq!(refs(ptr), 1);
-
-        drop(e1);
-        assert_eq!(refs(ptr), 0);
-    }
-
-    #[test]
-    fn test_replace() {
-        let cache = fifo(10);
-
-        insert_fifo(&cache, 114, "xx");
-        assert_eq!(cache.usage(), 2);
-
-        insert_fifo(&cache, 514, "QwQ");
-        assert_eq!(cache.usage(), 5);
-
-        insert_fifo(&cache, 114, "(0.0)");
-        assert_eq!(cache.usage(), 8);
-
-        assert_eq!(
-            cache.shards[0].lock().eviction.dump(),
-            vec![(514, "QwQ".to_string()), (114, "(0.0)".to_string())],
-        );
-    }
-
-    #[test]
-    fn test_replace_with_external_refs() {
-        let cache = fifo(10);
-
-        insert_fifo(&cache, 514, "QwQ");
-        insert_fifo(&cache, 114, "(0.0)");
-
-        let e4 = cache.get(&514).unwrap();
-        let e5 = insert_fifo(&cache, 514, "bili");
-
-        assert_eq!(e4.refs(), 1);
-        assert_eq!(e5.refs(), 1);
-
-        // remains: 514 => QwQ (3), 514 => bili (4)
-        // evicted: 114 => (0.0) (5)
-        assert_eq!(cache.usage(), 7);
-
-        assert!(cache.get(&114).is_none());
-        assert_eq!(cache.get(&514).unwrap().value(), "bili");
-        assert_eq!(e4.value(), "QwQ");
-
-        cache.remove(&514);
-        assert_eq!(e5.value(), "bili");
-
-        drop(e5);
-        assert!(cache.get(&514).is_none());
-        assert_eq!(e4.value(), "QwQ");
-
-        assert_eq!(cache.usage(), 3);
-        drop(e4);
-        assert_eq!(cache.usage(), 0);
-    }
-
-    #[test]
-    fn test_reinsert_while_all_referenced_lru() {
-        let cache = lru(10);
-
-        let e1 = insert_lru(&cache, 1, "111");
-        let e2 = insert_lru(&cache, 2, "222");
-        let e3 = insert_lru(&cache, 3, "333");
-        assert_eq!(cache.usage(), 9);
-
-        // No entry will be released because all of them are referenced externally.
-        let e4 = insert_lru(&cache, 4, "444");
-        assert_eq!(cache.usage(), 12);
-
-        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
-        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
-
-        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
-        drop(e1);
-        assert_eq!(cache.usage(), 9);
-
-        // `222` and `333` will be reinserted
-        drop(e2);
-        drop(e3);
-        assert_eq!(
-            cache.shards[0].lock().eviction.dump(),
-            vec![(4, "444".to_string()), (2, "222".to_string()), (3, "333".to_string()),]
-        );
-        assert_eq!(cache.usage(), 9);
-
-        // `444` will be reinserted
-        drop(e4);
-        assert_eq!(
-            cache.shards[0].lock().eviction.dump(),
-            vec![(2, "222".to_string()), (3, "333".to_string()), (4, "444".to_string()),]
-        );
-        assert_eq!(cache.usage(), 9);
-    }
-
-    #[test]
-    fn test_reinsert_while_all_referenced_fifo() {
-        let cache = fifo(10);
-
-        let e1 = insert_fifo(&cache, 1, "111");
-        let e2 = insert_fifo(&cache, 2, "222");
-        let e3 = insert_fifo(&cache, 3, "333");
-        assert_eq!(cache.usage(), 9);
-
-        // No entry will be released because all of them are referenced externally.
-        let e4 = insert_fifo(&cache, 4, "444");
-        assert_eq!(cache.usage(), 12);
-
-        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
-        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
-
-        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
-        drop(e1);
-        assert_eq!(cache.usage(), 9);
-
-        // `222` and `333` will be not reinserted because fifo will ignore reinsert operations.
-        drop([e2, e3, e4]);
-        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
-        assert_eq!(cache.usage(), 3);
-
-        // Note:
-        //
-        // For cache policy like FIFO, the entries will not be reinserted while all handles are referenced.
-        // It's okay for this is not a common situation and is not supposed to happen in real workload.
     }
 }
diff --git a/foyer-memory/src/context.rs b/foyer-memory/src/context.rs
index 041fcd78..3157a6f2 100644
--- a/foyer-memory/src/context.rs
+++ b/foyer-memory/src/context.rs
@@ -12,6 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum CacheContext {
     /// The default context shared by all eviction container implementations.
     Default,
@@ -25,6 +26,6 @@ impl Default for CacheContext {
 }
 
 /// The overhead of `Context` itself and the conversion should be light.
-pub trait Context: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static {}
+pub trait Context: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static + Clone {}
 
-impl<T> Context for T where T: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static {}
+impl<T> Context for T where T: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static + Clone {}
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index 208e5c06..c611fdef 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -25,7 +25,7 @@ use crate::{
     CacheContext, Key, Value,
 };
 
-#[derive(Debug, Default)]
+#[derive(Debug, Clone)]
 pub struct FifoContext;
 
 impl From<CacheContext> for FifoContext {
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index dbbfd4d0..e6dc8d3f 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -45,8 +45,7 @@ pub struct LfuConfig {
     pub cmsketch_eps: f64,
     pub cmsketch_confidence: f64,
 }
-
-#[derive(Debug, Default)]
+#[derive(Debug, Clone)]
 pub struct LfuContext;
 
 impl From<CacheContext> for LfuContext {
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index bf8c36cb..2608eafe 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -39,7 +39,7 @@ pub struct LruConfig {
     pub high_priority_pool_ratio: f64,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum LruContext {
     HighPriority,
     LowPriority,
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
new file mode 100644
index 00000000..328607b8
--- /dev/null
+++ b/foyer-memory/src/generic.rs
@@ -0,0 +1,949 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    future::Future,
+    hash::BuildHasher,
+    ops::Deref,
+    ptr::NonNull,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
+
+use ahash::RandomState;
+use crossbeam::queue::ArrayQueue;
+use futures::FutureExt;
+use hashbrown::hash_map::{Entry as HashMapEntry, HashMap};
+use itertools::Itertools;
+use parking_lot::Mutex;
+use tokio::{sync::oneshot, task::JoinHandle};
+
+use crate::{
+    eviction::Eviction, handle::Handle, indexer::Indexer, listener::CacheEventListener, metrics::Metrics, CacheContext,
+    Key, Value,
+};
+
+struct CacheSharedState<T, L> {
+    metrics: Metrics,
+    /// The object pool to avoid frequent handle allocating, shared by all shards.
+    object_pool: ArrayQueue<Box<T>>,
+    listener: L,
+}
+
+#[expect(clippy::type_complexity)]
+struct CacheShard<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    indexer: I,
+    eviction: E,
+
+    capacity: usize,
+    usage: Arc<AtomicUsize>,
+
+    waiters: HashMap<K, Vec<oneshot::Sender<GenericCacheEntry<K, V, H, E, I, L, S>>>>,
+
+    state: Arc<CacheSharedState<H, L>>,
+}
+
+impl<K, V, H, E, I, L, S> CacheShard<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn new(
+        capacity: usize,
+        eviction_config: &E::Config,
+        usage: Arc<AtomicUsize>,
+        context: Arc<CacheSharedState<H, L>>,
+    ) -> Self {
+        let indexer = I::new();
+        let eviction = unsafe { E::new(capacity, eviction_config) };
+        let waiters = HashMap::default();
+        Self {
+            indexer,
+            eviction,
+            capacity,
+            usage,
+            waiters,
+            state: context,
+        }
+    }
+
+    /// Insert a new entry into the cache. The handle for the new entry is returned.
+    unsafe fn insert(
+        &mut self,
+        hash: u64,
+        key: K,
+        value: V,
+        charge: usize,
+        context: H::Context,
+        last_reference_entries: &mut Vec<(K, V, H::Context, usize)>,
+    ) -> NonNull<H> {
+        let mut handle = self.state.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
+        handle.init(hash, key, value, charge, context);
+        let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
+
+        self.evict(charge, last_reference_entries);
+
+        debug_assert!(!ptr.as_ref().base().is_in_indexer());
+        if let Some(old) = self.indexer.insert(ptr) {
+            self.state.metrics.replace.fetch_add(1, Ordering::Relaxed);
+
+            debug_assert!(!old.as_ref().base().is_in_indexer());
+            if old.as_ref().base().is_in_eviction() {
+                self.eviction.remove(old);
+            }
+            debug_assert!(!old.as_ref().base().is_in_eviction());
+            // Because the `old` handle is removed from the indexer, it will not be reinserted again.
+            if let Some(entry) = self.try_release_handle(old, false) {
+                last_reference_entries.push(entry);
+            }
+        } else {
+            self.state.metrics.insert.fetch_add(1, Ordering::Relaxed);
+        }
+        self.eviction.push(ptr);
+
+        debug_assert!(ptr.as_ref().base().is_in_indexer());
+        debug_assert!(ptr.as_ref().base().is_in_indexer());
+
+        self.usage.fetch_add(charge, Ordering::Relaxed);
+        ptr.as_mut().base_mut().inc_refs();
+
+        ptr
+    }
+
+    unsafe fn get(&mut self, hash: u64, key: &K) -> Option<NonNull<H>> {
+        let mut ptr = match self.indexer.get(hash, key) {
+            Some(ptr) => {
+                self.state.metrics.hit.fetch_add(1, Ordering::Relaxed);
+                ptr
+            }
+            None => {
+                self.state.metrics.miss.fetch_add(1, Ordering::Relaxed);
+                return None;
+            }
+        };
+        let base = ptr.as_mut().base_mut();
+        debug_assert!(base.is_in_indexer());
+
+        base.inc_refs();
+        self.eviction.access(ptr);
+
+        Some(ptr)
+    }
+
+    /// Remove a key from the cache.
+    ///
+    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
+    unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V, H::Context, usize)> {
+        let ptr = self.indexer.remove(hash, key)?;
+        self.state.metrics.remove.fetch_add(1, Ordering::Relaxed);
+        if ptr.as_ref().base().is_in_eviction() {
+            self.eviction.remove(ptr);
+        }
+        debug_assert!(!ptr.as_ref().base().is_in_indexer());
+        debug_assert!(!ptr.as_ref().base().is_in_eviction());
+        self.try_release_handle(ptr, false)
+    }
+
+    /// Clear all cache entries.
+    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
+        // TODO(MrCroxx): Avoid collecting here?
+        let ptrs = self.indexer.drain().collect_vec();
+        let eptrs = self.eviction.clear();
+
+        // Assert that the handles in the indexer covers the handles in the eviction container.
+        if cfg!(debug_assertions) {
+            use std::{collections::HashSet as StdHashSet, hash::RandomState as StdRandomState};
+            let ptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(ptrs.iter().copied());
+            let eptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(eptrs.iter().copied());
+            debug_assert!((&eptrs - &ptrs).is_empty());
+        }
+
+        self.state.metrics.remove.fetch_add(ptrs.len(), Ordering::Relaxed);
+
+        // The handles in the indexer covers the handles in the eviction container.
+        // So only the handles drained from the indexer need to be released.
+        for ptr in ptrs {
+            debug_assert!(!ptr.as_ref().base().is_in_indexer());
+            if let Some(entry) = self.try_release_handle(ptr, false) {
+                last_reference_entries.push(entry);
+            }
+        }
+    }
+
+    unsafe fn evict(&mut self, charge: usize, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
+        while self.usage.load(Ordering::Relaxed) + charge > self.capacity
+            && let Some(evicted) = self.eviction.pop()
+        {
+            self.state.metrics.evict.fetch_add(1, Ordering::Relaxed);
+            let base = evicted.as_ref().base();
+            debug_assert!(base.is_in_indexer());
+            debug_assert!(!base.is_in_eviction());
+            if let Some(entry) = self.try_release_handle(evicted, false) {
+                last_reference_entries.push(entry);
+            }
+        }
+    }
+
+    /// Release a handle used by an external user.
+    ///
+    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
+    unsafe fn try_release_external_handle(&mut self, mut ptr: NonNull<H>) -> Option<(K, V, H::Context, usize)> {
+        ptr.as_mut().base_mut().dec_refs();
+        self.try_release_handle(ptr, true)
+    }
+
+    /// Try release handle if there is no external reference and no reinsertion is needed.
+    ///
+    /// Return the entry if the handle is released.
+    ///
+    /// Recycle it if possible.
+    unsafe fn try_release_handle(&mut self, mut ptr: NonNull<H>, reinsert: bool) -> Option<(K, V, H::Context, usize)> {
+        let base = ptr.as_mut().base_mut();
+
+        if base.has_refs() {
+            return None;
+        }
+
+        debug_assert!(base.is_inited());
+        debug_assert!(!base.has_refs());
+
+        // If the entry is not updated or removed from the cache, try to reinsert it or remove it from the indexer and
+        // the eviction container.
+        if base.is_in_indexer() {
+            // The usage is higher than the capacity means most handles are held externally,
+            // the cache shard cannot release enough charges for the new inserted entries.
+            // In this case, the reinsertion should be given up.
+            if reinsert && self.usage.load(Ordering::Relaxed) <= self.capacity {
+                let was_in_eviction = base.is_in_eviction();
+                self.eviction.reinsert(ptr);
+                if ptr.as_ref().base().is_in_eviction() {
+                    if was_in_eviction {
+                        self.state.metrics.reinsert.fetch_add(1, Ordering::Relaxed);
+                    }
+                    return None;
+                }
+            }
+
+            // If the entry has not been reinserted, remove it from the indexer and the eviction container (if needed).
+            self.indexer.remove(base.hash(), base.key());
+            if ptr.as_ref().base().is_in_eviction() {
+                self.eviction.remove(ptr);
+            }
+        }
+
+        // Here the handle is neither in the indexer nor in the eviction container.
+        debug_assert!(!base.is_in_indexer());
+        debug_assert!(!base.is_in_eviction());
+        debug_assert!(!base.has_refs());
+
+        self.state.metrics.release.fetch_add(1, Ordering::Relaxed);
+
+        self.usage.fetch_sub(base.charge(), Ordering::Relaxed);
+        let entry = base.take();
+
+        let handle = Box::from_raw(ptr.as_ptr());
+        let _ = self.state.object_pool.push(handle);
+
+        Some(entry)
+    }
+}
+
+impl<K, V, H, E, I, L, S> Drop for CacheShard<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn drop(&mut self) {
+        unsafe { self.clear(&mut vec![]) }
+    }
+}
+
+pub struct CacheConfig<E, L, S = RandomState>
+where
+    E: Eviction,
+    L: CacheEventListener<<E::Handle as Handle>::Key, <E::Handle as Handle>::Value>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub capacity: usize,
+    pub shards: usize,
+    pub eviction_config: E::Config,
+    pub object_pool_capacity: usize,
+    pub hash_builder: S,
+    pub event_listener: L,
+}
+
+#[expect(clippy::type_complexity)]
+pub enum GenericEntry<K, V, H, E, I, L, S, ER>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+    ER: std::error::Error,
+{
+    Invalid,
+    Hit(GenericCacheEntry<K, V, H, E, I, L, S>),
+    Wait(oneshot::Receiver<GenericCacheEntry<K, V, H, E, I, L, S>>),
+    Miss(JoinHandle<std::result::Result<GenericCacheEntry<K, V, H, E, I, L, S>, ER>>),
+}
+
+impl<K, V, H, E, I, L, S, ER> Default for GenericEntry<K, V, H, E, I, L, S, ER>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+    ER: std::error::Error,
+{
+    fn default() -> Self {
+        Self::Invalid
+    }
+}
+
+impl<K, V, H, E, I, L, S, ER> Future for GenericEntry<K, V, H, E, I, L, S, ER>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+    ER: std::error::Error + From<oneshot::error::RecvError>,
+{
+    type Output = std::result::Result<GenericCacheEntry<K, V, H, E, I, L, S>, ER>;
+
+    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
+        match &mut *self {
+            Self::Invalid => unreachable!(),
+            Self::Hit(_) => std::task::Poll::Ready(Ok(match std::mem::take(&mut *self) {
+                GenericEntry::Hit(entry) => entry,
+                _ => unreachable!(),
+            })),
+            Self::Wait(waiter) => waiter.poll_unpin(cx).map_err(|err| err.into()),
+            Self::Miss(join_handle) => join_handle.poll_unpin(cx).map(|join_result| join_result.unwrap()),
+        }
+    }
+}
+
+#[expect(clippy::type_complexity)]
+pub struct GenericCache<K, V, H, E, I, L, S = RandomState>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    shards: Vec<Mutex<CacheShard<K, V, H, E, I, L, S>>>,
+
+    capacity: usize,
+    usages: Vec<Arc<AtomicUsize>>,
+
+    context: Arc<CacheSharedState<H, L>>,
+
+    hash_builder: S,
+}
+
+impl<K, V, H, E, I, L, S> GenericCache<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn new(config: CacheConfig<E, L, S>) -> Self {
+        let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
+        let context = Arc::new(CacheSharedState {
+            metrics: Metrics::default(),
+            object_pool: ArrayQueue::new(config.object_pool_capacity),
+            listener: config.event_listener,
+        });
+
+        let shard_capacity = config.capacity / config.shards;
+
+        let shards = usages
+            .iter()
+            .map(|usage| CacheShard::new(shard_capacity, &config.eviction_config, usage.clone(), context.clone()))
+            .map(Mutex::new)
+            .collect_vec();
+
+        Self {
+            shards,
+            capacity: config.capacity,
+            usages,
+            context,
+            hash_builder: config.hash_builder,
+        }
+    }
+
+    pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> GenericCacheEntry<K, V, H, E, I, L, S> {
+        self.insert_with_context(key, value, charge, CacheContext::default())
+    }
+
+    pub fn insert_with_context(
+        self: &Arc<Self>,
+        key: K,
+        value: V,
+        charge: usize,
+        context: CacheContext,
+    ) -> GenericCacheEntry<K, V, H, E, I, L, S> {
+        let hash = self.hash_builder.hash_one(&key);
+
+        let mut to_deallocate = vec![];
+
+        let (entry, waiters) = unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            let waiters = shard.waiters.remove(&key);
+            let mut ptr = shard.insert(hash, key, value, charge, context.into(), &mut to_deallocate);
+            if let Some(waiters) = waiters.as_ref() {
+                ptr.as_mut().base_mut().inc_refs_by(waiters.len());
+            }
+            let entry = GenericCacheEntry {
+                cache: self.clone(),
+                ptr,
+            };
+            (entry, waiters)
+        };
+
+        if let Some(waiters) = waiters {
+            for waiter in waiters {
+                let _ = waiter.send(GenericCacheEntry {
+                    cache: self.clone(),
+                    ptr: entry.ptr,
+                });
+            }
+        }
+
+        // Do not deallocate data within the lock section.
+        for (key, value, context, charges) in to_deallocate {
+            self.context.listener.on_release(key, value, context.into(), charges)
+        }
+
+        entry
+    }
+
+    pub fn remove(&self, key: &K) {
+        let hash = self.hash_builder.hash_one(key);
+
+        let entry = unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            shard.remove(hash, key)
+        };
+
+        // Do not deallocate data within the lock section.
+        if let Some((key, value, context, charges)) = entry {
+            self.context.listener.on_release(key, value, context.into(), charges);
+        }
+    }
+
+    pub fn get(self: &Arc<Self>, key: &K) -> Option<GenericCacheEntry<K, V, H, E, I, L, S>> {
+        let hash = self.hash_builder.hash_one(key);
+
+        unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            shard.get(hash, key).map(|ptr| GenericCacheEntry {
+                cache: self.clone(),
+                ptr,
+            })
+        }
+    }
+
+    pub fn clear(&self) {
+        let mut to_deallocate = vec![];
+        for shard in self.shards.iter() {
+            let mut shard = shard.lock();
+            unsafe { shard.clear(&mut to_deallocate) };
+        }
+    }
+
+    pub fn capacity(&self) -> usize {
+        self.capacity
+    }
+
+    pub fn usage(&self) -> usize {
+        self.usages.iter().map(|usage| usage.load(Ordering::Relaxed)).sum()
+    }
+
+    pub fn metrics(&self) -> &Metrics {
+        &self.context.metrics
+    }
+
+    unsafe fn try_release_external_handle(&self, ptr: NonNull<H>) {
+        let entry = {
+            let base = ptr.as_ref().base();
+            let mut shard = self.shards[base.hash() as usize % self.shards.len()].lock();
+            shard.try_release_external_handle(ptr)
+        };
+
+        // Do not deallocate data within the lock section.
+        if let Some((key, value, context, charges)) = entry {
+            self.context.listener.on_release(key, value, context.into(), charges);
+        }
+    }
+}
+
+// TODO(MrCroxx): use `hashbrown::HashTable` with `Handle` may relax the `Clone` bound?
+impl<K, V, H, E, I, L, S> GenericCache<K, V, H, E, I, L, S>
+where
+    K: Key + Clone,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> GenericEntry<K, V, H, E, I, L, S, ER>
+    where
+        F: FnOnce() -> FU,
+        FU: Future<Output = std::result::Result<(V, usize, CacheContext), ER>> + Send + 'static,
+        ER: std::error::Error + Send + 'static,
+    {
+        let hash = self.hash_builder.hash_one(&key);
+
+        unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            if let Some(ptr) = shard.get(hash, &key) {
+                return GenericEntry::Hit(GenericCacheEntry {
+                    cache: self.clone(),
+                    ptr,
+                });
+            }
+            let entry = match shard.waiters.entry(key.clone()) {
+                HashMapEntry::Occupied(mut o) => {
+                    let (tx, rx) = oneshot::channel();
+                    o.get_mut().push(tx);
+                    GenericEntry::Wait(rx)
+                }
+                HashMapEntry::Vacant(v) => {
+                    v.insert(vec![]);
+                    let cache = self.clone();
+                    let future = f();
+                    let join = tokio::spawn(async move {
+                        let (value, charge, context) = match future.await {
+                            Ok((value, charge, context)) => (value, charge, context),
+                            Err(e) => {
+                                let mut shard = cache.shards[hash as usize % cache.shards.len()].lock();
+                                shard.waiters.remove(&key);
+                                return Err(e);
+                            }
+                        };
+                        let entry = cache.insert_with_context(key, value, charge, context);
+                        Ok(entry)
+                    });
+                    GenericEntry::Miss(join)
+                }
+            };
+            match entry {
+                GenericEntry::Wait(_) => shard.state.metrics.queue.fetch_add(1, Ordering::Relaxed),
+                GenericEntry::Miss(_) => shard.state.metrics.fetch.fetch_add(1, Ordering::Relaxed),
+                _ => unreachable!(),
+            };
+            entry
+        }
+    }
+}
+
+pub struct GenericCacheEntry<K, V, H, E, I, L, S = RandomState>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    cache: Arc<GenericCache<K, V, H, E, I, L, S>>,
+    ptr: NonNull<H>,
+}
+
+impl<K, V, H, E, I, L, S> GenericCacheEntry<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn key(&self) -> &H::Key {
+        unsafe { self.ptr.as_ref().base().key() }
+    }
+
+    pub fn value(&self) -> &H::Value {
+        unsafe { self.ptr.as_ref().base().value() }
+    }
+
+    pub fn context(&self) -> &H::Context {
+        unsafe { self.ptr.as_ref().base().context() }
+    }
+
+    pub fn charge(&self) -> usize {
+        unsafe { self.ptr.as_ref().base().charge() }
+    }
+
+    pub fn refs(&self) -> usize {
+        unsafe { self.ptr.as_ref().base().refs() }
+    }
+}
+
+impl<K, V, H, E, I, L, S> Clone for GenericCacheEntry<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn clone(&self) -> Self {
+        let mut ptr = self.ptr;
+
+        unsafe {
+            let base = ptr.as_mut().base_mut();
+            debug_assert!(base.has_refs());
+            base.inc_refs();
+        }
+
+        Self {
+            cache: self.cache.clone(),
+            ptr,
+        }
+    }
+}
+
+impl<K, V, H, E, I, L, S> Drop for GenericCacheEntry<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn drop(&mut self) {
+        unsafe { self.cache.try_release_external_handle(self.ptr) }
+    }
+}
+
+impl<K, V, H, E, I, L, S> Deref for GenericCacheEntry<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    type Target = V;
+
+    fn deref(&self) -> &Self::Target {
+        self.value()
+    }
+}
+
+unsafe impl<K, V, H, E, I, L, S> Send for GenericCacheEntry<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+}
+unsafe impl<K, V, H, E, I, L, S> Sync for GenericCacheEntry<K, V, H, E, I, L, S>
+where
+    K: Key,
+    V: Value,
+    H: Handle<Key = K, Value = V>,
+    E: Eviction<Handle = H>,
+    I: Indexer<Key = K, Handle = H>,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+}
+
+#[cfg(test)]
+mod tests {
+    use rand::{rngs::SmallRng, RngCore, SeedableRng};
+
+    use super::*;
+    use crate::{
+        cache::{FifoCache, FifoCacheConfig, FifoCacheEntry, LruCache, LruCacheConfig, LruCacheEntry},
+        eviction::{
+            fifo::{FifoConfig, FifoHandle},
+            lru::LruConfig,
+            test_utils::TestEviction,
+        },
+        listener::DefaultCacheEventListener,
+    };
+
+    fn is_send_sync_static<T: Send + Sync + 'static>() {}
+
+    #[test]
+    fn test_send_sync_static() {
+        is_send_sync_static::<FifoCache<(), ()>>();
+        is_send_sync_static::<FifoCacheConfig<(), ()>>();
+        is_send_sync_static::<LruCache<(), ()>>();
+        is_send_sync_static::<LruCacheConfig<(), ()>>();
+    }
+
+    #[test]
+    fn test_cache_fuzzy() {
+        const CAPACITY: usize = 256;
+
+        let config = FifoCacheConfig {
+            capacity: CAPACITY,
+            shards: 4,
+            eviction_config: FifoConfig {},
+            object_pool_capacity: 16,
+            hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
+        };
+        let cache = Arc::new(FifoCache::<u64, u64>::new(config));
+
+        let mut rng = SmallRng::seed_from_u64(114514);
+        for _ in 0..100000 {
+            let key = rng.next_u64();
+            if let Some(entry) = cache.get(&key) {
+                assert_eq!(key, *entry);
+                drop(entry);
+                continue;
+            }
+            cache.insert(key, key, 1);
+        }
+        assert_eq!(cache.usage(), CAPACITY);
+    }
+
+    fn fifo(capacity: usize) -> Arc<FifoCache<u64, String>> {
+        let config = FifoCacheConfig {
+            capacity,
+            shards: 1,
+            eviction_config: FifoConfig {},
+            object_pool_capacity: 1,
+            hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
+        };
+        Arc::new(FifoCache::<u64, String>::new(config))
+    }
+
+    fn lru(capacity: usize) -> Arc<LruCache<u64, String>> {
+        let config = LruCacheConfig {
+            capacity,
+            shards: 1,
+            eviction_config: LruConfig {
+                high_priority_pool_ratio: 0.0,
+            },
+            object_pool_capacity: 1,
+            hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
+        };
+        Arc::new(LruCache::<u64, String>::new(config))
+    }
+
+    fn insert_fifo(cache: &Arc<FifoCache<u64, String>>, key: u64, value: &str) -> FifoCacheEntry<u64, String> {
+        cache.insert(key, value.to_string(), value.len())
+    }
+
+    fn insert_lru(cache: &Arc<LruCache<u64, String>>, key: u64, value: &str) -> LruCacheEntry<u64, String> {
+        cache.insert(key, value.to_string(), value.len())
+    }
+
+    #[test]
+    fn test_reference_count() {
+        let cache = fifo(100);
+
+        let refs = |ptr: NonNull<FifoHandle<u64, String>>| unsafe { ptr.as_ref().base().refs() };
+
+        let e1 = insert_fifo(&cache, 42, "the answer to life, the universe, and everything");
+        let ptr = e1.ptr;
+        assert_eq!(refs(ptr), 1);
+
+        let e2 = cache.get(&42).unwrap();
+        assert_eq!(refs(ptr), 2);
+
+        let e3 = e2.clone();
+        assert_eq!(refs(ptr), 3);
+
+        drop(e2);
+        assert_eq!(refs(ptr), 2);
+
+        drop(e3);
+        assert_eq!(refs(ptr), 1);
+
+        drop(e1);
+        assert_eq!(refs(ptr), 0);
+    }
+
+    #[test]
+    fn test_replace() {
+        let cache = fifo(10);
+
+        insert_fifo(&cache, 114, "xx");
+        assert_eq!(cache.usage(), 2);
+
+        insert_fifo(&cache, 514, "QwQ");
+        assert_eq!(cache.usage(), 5);
+
+        insert_fifo(&cache, 114, "(0.0)");
+        assert_eq!(cache.usage(), 8);
+
+        assert_eq!(
+            cache.shards[0].lock().eviction.dump(),
+            vec![(514, "QwQ".to_string()), (114, "(0.0)".to_string())],
+        );
+    }
+
+    #[test]
+    fn test_replace_with_external_refs() {
+        let cache = fifo(10);
+
+        insert_fifo(&cache, 514, "QwQ");
+        insert_fifo(&cache, 114, "(0.0)");
+
+        let e4 = cache.get(&514).unwrap();
+        let e5 = insert_fifo(&cache, 514, "bili");
+
+        assert_eq!(e4.refs(), 1);
+        assert_eq!(e5.refs(), 1);
+
+        // remains: 514 => QwQ (3), 514 => bili (4)
+        // evicted: 114 => (0.0) (5)
+        assert_eq!(cache.usage(), 7);
+
+        assert!(cache.get(&114).is_none());
+        assert_eq!(cache.get(&514).unwrap().value(), "bili");
+        assert_eq!(e4.value(), "QwQ");
+
+        cache.remove(&514);
+        assert_eq!(e5.value(), "bili");
+
+        drop(e5);
+        assert!(cache.get(&514).is_none());
+        assert_eq!(e4.value(), "QwQ");
+
+        assert_eq!(cache.usage(), 3);
+        drop(e4);
+        assert_eq!(cache.usage(), 0);
+    }
+
+    #[test]
+    fn test_reinsert_while_all_referenced_lru() {
+        let cache = lru(10);
+
+        let e1 = insert_lru(&cache, 1, "111");
+        let e2 = insert_lru(&cache, 2, "222");
+        let e3 = insert_lru(&cache, 3, "333");
+        assert_eq!(cache.usage(), 9);
+
+        // No entry will be released because all of them are referenced externally.
+        let e4 = insert_lru(&cache, 4, "444");
+        assert_eq!(cache.usage(), 12);
+
+        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
+        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
+
+        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
+        drop(e1);
+        assert_eq!(cache.usage(), 9);
+
+        // `222` and `333` will be reinserted
+        drop(e2);
+        drop(e3);
+        assert_eq!(
+            cache.shards[0].lock().eviction.dump(),
+            vec![(4, "444".to_string()), (2, "222".to_string()), (3, "333".to_string()),]
+        );
+        assert_eq!(cache.usage(), 9);
+
+        // `444` will be reinserted
+        drop(e4);
+        assert_eq!(
+            cache.shards[0].lock().eviction.dump(),
+            vec![(2, "222".to_string()), (3, "333".to_string()), (4, "444".to_string()),]
+        );
+        assert_eq!(cache.usage(), 9);
+    }
+
+    #[test]
+    fn test_reinsert_while_all_referenced_fifo() {
+        let cache = fifo(10);
+
+        let e1 = insert_fifo(&cache, 1, "111");
+        let e2 = insert_fifo(&cache, 2, "222");
+        let e3 = insert_fifo(&cache, 3, "333");
+        assert_eq!(cache.usage(), 9);
+
+        // No entry will be released because all of them are referenced externally.
+        let e4 = insert_fifo(&cache, 4, "444");
+        assert_eq!(cache.usage(), 12);
+
+        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
+        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
+
+        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
+        drop(e1);
+        assert_eq!(cache.usage(), 9);
+
+        // `222` and `333` will be not reinserted because fifo will ignore reinsert operations.
+        drop([e2, e3, e4]);
+        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
+        assert_eq!(cache.usage(), 3);
+
+        // Note:
+        //
+        // For cache policy like FIFO, the entries will not be reinserted while all handles are referenced.
+        // It's okay for this is not a common situation and is not supposed to happen in real workload.
+    }
+}
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 0b026735..c4397451 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -76,6 +76,7 @@ impl<T: Send + Sync + 'static> Value for T {}
 mod cache;
 mod context;
 mod eviction;
+mod generic;
 mod handle;
 mod indexer;
 mod listener;
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index 6795618a..a909f2b0 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -12,44 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use ahash::RandomState;
-
-use crate::{
-    cache::{Cache, CacheConfig, CacheEntry},
-    eviction::{
-        fifo::{Fifo, FifoHandle},
-        lfu::{Lfu, LfuHandle},
-        lru::{Lru, LruHandle},
-    },
-    indexer::HashTableIndexer,
-    listener::DefaultCacheEventListener,
-};
-
-pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    Cache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
-pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Fifo<K, V>, L, S>;
-pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    CacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
-
-pub type LruCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    Cache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
-pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lru<K, V>, L, S>;
-pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    CacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
-
-pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    Cache<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
-pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lfu<K, V>, L, S>;
-pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    CacheEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
-
 pub use crate::{
-    cache::Entry,
+    cache::{Cache, CacheEntry, Entry, EntryState, FifoCacheConfig, LfuCacheConfig, LruCacheConfig},
     context::CacheContext,
-    eviction::{
-        fifo::{FifoConfig, FifoContext},
-        lfu::{LfuConfig, LfuContext},
-        lru::{LruConfig, LruContext},
-    },
+    eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig},
     listener::CacheEventListener,
+    metrics::Metrics,
 };

From 52cb3b788c1e7706cedb6bcd4d5361204a3daa18 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 12 Mar 2024 13:44:07 +0800
Subject: [PATCH 228/261] test: add test for unified cache (#289)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs | 136 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 135 insertions(+), 1 deletion(-)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 1c902b18..7f3786d0 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -186,7 +186,7 @@ where
     }
 }
 
-pub enum Cache<K, V, L, S = RandomState>
+pub enum Cache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState>
 where
     K: Key,
     V: Value,
@@ -426,3 +426,137 @@ where
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::{ops::Range, time::Duration};
+
+    use futures::future::join_all;
+    use itertools::Itertools;
+    use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng};
+
+    use super::*;
+    use crate::{FifoConfig, LfuConfig, LruConfig};
+
+    const CAPACITY: usize = 100;
+    const SHARDS: usize = 4;
+    const OBJECT_POOL_CAPACITY: usize = 64;
+    const RANGE: Range<u64> = 0..1000;
+    const OPS: usize = 10000;
+    const CONCURRENCY: usize = 8;
+
+    fn fifo() -> Cache<u64, u64> {
+        Cache::fifo(FifoCacheConfig {
+            capacity: CAPACITY,
+            shards: SHARDS,
+            eviction_config: FifoConfig {},
+            object_pool_capacity: OBJECT_POOL_CAPACITY,
+            hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
+        })
+    }
+
+    fn lru() -> Cache<u64, u64> {
+        Cache::lru(LruCacheConfig {
+            capacity: CAPACITY,
+            shards: SHARDS,
+            eviction_config: LruConfig {
+                high_priority_pool_ratio: 0.1,
+            },
+            object_pool_capacity: OBJECT_POOL_CAPACITY,
+            hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
+        })
+    }
+
+    fn lfu() -> Cache<u64, u64> {
+        Cache::lfu(LfuCacheConfig {
+            capacity: CAPACITY,
+            shards: SHARDS,
+            eviction_config: LfuConfig {
+                window_capacity_ratio: 0.1,
+                protected_capacity_ratio: 0.8,
+                cmsketch_eps: 0.001,
+                cmsketch_confidence: 0.9,
+            },
+            object_pool_capacity: OBJECT_POOL_CAPACITY,
+            hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
+        })
+    }
+
+    fn init_cache(cache: &Cache<u64, u64>, rng: &mut StdRng) {
+        let mut v = RANGE.collect_vec();
+        v.shuffle(rng);
+        for i in v {
+            cache.insert(i, i, 1);
+        }
+    }
+
+    async fn operate(cache: &Cache<u64, u64>, rng: &mut StdRng) {
+        let i = rng.gen_range(RANGE);
+        match rng.gen_range(0..=3) {
+            0 => {
+                let entry = cache.insert(i, i, 1);
+                assert_eq!(*entry.key(), i);
+                assert_eq!(entry.key(), entry.value());
+            }
+            1 => {
+                if let Some(entry) = cache.get(&i) {
+                    assert_eq!(*entry.key(), i);
+                    assert_eq!(entry.key(), entry.value());
+                }
+            }
+            2 => {
+                cache.remove(&i);
+            }
+            3 => {
+                let entry = cache
+                    .entry(i, || async move {
+                        tokio::time::sleep(Duration::from_micros(10)).await;
+                        Ok::<_, tokio::sync::oneshot::error::RecvError>((i, 1, CacheContext::Default))
+                    })
+                    .await
+                    .unwrap();
+                assert_eq!(*entry.key(), i);
+                assert_eq!(entry.key(), entry.value());
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    async fn case(cache: Cache<u64, u64>) {
+        let mut rng = StdRng::seed_from_u64(42);
+
+        init_cache(&cache, &mut rng);
+
+        let handles = (0..CONCURRENCY)
+            .map(|_| {
+                let cache = cache.clone();
+                let mut rng = rng.clone();
+                tokio::spawn(async move {
+                    for _ in 0..OPS {
+                        operate(&cache, &mut rng).await;
+                    }
+                })
+            })
+            .collect_vec();
+
+        join_all(handles).await;
+    }
+
+    #[tokio::test]
+    async fn test_fifo_cache() {
+        case(fifo()).await
+    }
+
+    #[tokio::test]
+    async fn test_lru_cache() {
+        case(lru()).await
+    }
+
+    #[tokio::test]
+    async fn test_lfu_cache() {
+        case(lfu()).await
+    }
+}

From 0937254b60543bf85479b2232ab38ed50d5387a3 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 12 Mar 2024 14:53:26 +0800
Subject: [PATCH 229/261] chore: bump version and release foyer 0.6.0 (#290)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md                        | 21 +++++++++++++++++++++
 foyer-common/Cargo.toml             |  4 ++--
 foyer-experimental-bench/Cargo.toml |  8 ++++----
 foyer-experimental/Cargo.toml       |  4 ++--
 foyer-intrusive/Cargo.toml          |  6 +++---
 foyer-memory/Cargo.toml             |  4 ++--
 foyer-storage-bench/Cargo.toml      | 10 +++++-----
 foyer-storage/Cargo.toml            |  8 ++++----
 foyer-workspace-hack/Cargo.toml     |  2 +-
 foyer/Cargo.toml                    | 10 +++++-----
 10 files changed, 49 insertions(+), 28 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f3e5fb0..8f3086a1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,24 @@
+## 2024-03-12
+
+| crate | version |
+| - | - |
+| foyer | 0.6.0 |
+| foyer-common | 0.4.0 |
+| foyer-intrusive | 0.3.0 |
+| foyer-memory | 0.1.0 |
+| foyer-storage | 0.5.0 |
+| foyer-storage-bench | 0.5.0 |
+| foyer-workspace-hack | 0.3.0 |
+
+<details>
+
+### Changes
+
+- Release foyer in-memory cache as crate `foyer-memory`.
+- Bump other components with changes.
+
+</details>
+
 ## 2023-12-28
 
 | crate | version |
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 525a4393..16a298b5 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-common"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "common utils for foyer - the hybrid cache for Rust"
@@ -15,7 +15,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 anyhow = "1.0"
 bytes = "1"
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
diff --git a/foyer-experimental-bench/Cargo.toml b/foyer-experimental-bench/Cargo.toml
index f8cc3b81..6d368527 100644
--- a/foyer-experimental-bench/Cargo.toml
+++ b/foyer-experimental-bench/Cargo.toml
@@ -18,11 +18,11 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
-foyer-common = { version = "0.3", path = "../foyer-common" }
+foyer-common = { version = "0.4", path = "../foyer-common" }
 foyer-experimental = { version = "0.1", path = "../foyer-experimental" }
-foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.4", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.5", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
 http-body-util = "0.1"
diff --git a/foyer-experimental/Cargo.toml b/foyer-experimental/Cargo.toml
index 0101ae68..538389c6 100644
--- a/foyer-experimental/Cargo.toml
+++ b/foyer-experimental/Cargo.toml
@@ -16,7 +16,7 @@ normal = ["foyer-workspace-hack"]
 anyhow = "1.0"
 bytes = "1"
 crossbeam = { version = "0.8", features = ["std", "crossbeam-channel"] }
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 prometheus = "0.13"
@@ -27,7 +27,7 @@ tracing = "0.1"
 [dev-dependencies]
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
-foyer-common = { version = "0.3", path = "../foyer-common" }
+foyer-common = { version = "0.4", path = "../foyer-common" }
 hdrhistogram = "7"
 itertools = "0.12"
 rand = "0.8.5"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 87c89468..2d2618b4 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-intrusive"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "intrusive data structures for foyer - the hybrid cache for Rust"
@@ -15,8 +15,8 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { version = "0.3", path = "../foyer-common" }
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.4", path = "../foyer-common" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 memoffset = "0.9"
 parking_lot = "0.12"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index e3feee65..a2561e5f 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -17,8 +17,8 @@ ahash = "0.8"
 bitflags = "2"
 cmsketch = "0.2"
 crossbeam = "0.8"
-foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hashbrown = "0.14"
 itertools = "0.12"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index b0dc51b6..6a4e9331 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage-bench"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine bench tool for foyer - the hybrid cache for Rust"
@@ -17,10 +17,10 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
-foyer-common = { version = "0.3", path = "../foyer-common" }
-foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.4", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.4", path = "../foyer-common" }
+foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.5", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
 http-body-util = "0.1"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 69b91a79..8e5b485a 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
@@ -17,9 +17,9 @@ anyhow = "1.0"
 bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
-foyer-common = { version = "0.3", path = "../foyer-common" }
-foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.4", path = "../foyer-common" }
+foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.12"
 libc = "0.2"
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 76dac42e..1cc8e191 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -4,7 +4,7 @@
 
 [package]
 name = "foyer-workspace-hack"
-version = "0.2.0"
+version = "0.3.0"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "workspace-hack package, managed by hakari"
 license = "Apache-2.0"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index e3dc1741..ec556e48 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -13,8 +13,8 @@ homepage = "https://github.com/mrcroxx/foyer"
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
-foyer-common = { version = "0.3", path = "../foyer-common" }
-foyer-intrusive = { version = "0.2", path = "../foyer-intrusive" }
+foyer-common = { version = "0.4", path = "../foyer-common" }
+foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
 foyer-memory = { version = "0.1", path = "../foyer-memory" }
-foyer-storage = { version = "0.4", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.2", path = "../foyer-workspace-hack" }
+foyer-storage = { version = "0.5", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }

From 12eff6520b4eb24e8e61ebfff4d0d8d5271ee812 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 12 Mar 2024 18:41:03 +0800
Subject: [PATCH 230/261] fix: fix build with trace future (#291)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-experimental-bench/src/lib.rs | 7 ++++---
 foyer-storage-bench/src/main.rs     | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/foyer-experimental-bench/src/lib.rs b/foyer-experimental-bench/src/lib.rs
index c23b1788..cbaf3f27 100644
--- a/foyer-experimental-bench/src/lib.rs
+++ b/foyer-experimental-bench/src/lib.rs
@@ -35,7 +35,7 @@ pub fn init_logger() {
 #[cfg(feature = "trace")]
 pub fn init_logger() {
     use opentelemetry_sdk::{
-        trace::{BatchConfig, Config},
+        trace::{BatchConfigBuilder, Config},
         Resource,
     };
     use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
@@ -46,10 +46,11 @@ pub fn init_logger() {
         SERVICE_NAME,
         "foyer-storage-bench",
     )]));
-    let batch_config = BatchConfig::default()
+    let batch_config = BatchConfigBuilder::default()
         .with_max_queue_size(1048576)
         .with_max_export_batch_size(4096)
-        .with_max_concurrent_exports(4);
+        .with_max_concurrent_exports(4)
+        .build();
 
     let tracer = opentelemetry_otlp::new_pipeline()
         .tracing()
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index fc019f9f..d95fb6ea 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -432,7 +432,7 @@ fn init_logger() {
 #[cfg(feature = "trace")]
 fn init_logger() {
     use opentelemetry_sdk::{
-        trace::{BatchConfig, Config},
+        trace::{BatchConfigBuilder, Config},
         Resource,
     };
     use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
@@ -443,10 +443,11 @@ fn init_logger() {
         SERVICE_NAME,
         "foyer-storage-bench",
     )]));
-    let batch_config = BatchConfig::default()
+    let batch_config = BatchConfigBuilder::default()
         .with_max_queue_size(1048576)
         .with_max_export_batch_size(4096)
-        .with_max_concurrent_exports(4);
+        .with_max_concurrent_exports(4)
+        .build();
 
     let tracer = opentelemetry_otlp::new_pipeline()
         .tracing()

From bd3fc1af11a00517b5ad0cd44054a6f1b8e78bba Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 13 Mar 2024 11:47:51 +0800
Subject: [PATCH 231/261] chore: bump bench tools version (#292)

* chore: bump bench tools version

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update change log, restore unreleased version

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md                   | 14 ++++++++++++++
 foyer-storage-bench/Cargo.toml |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8f3086a1..6c88f014 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,17 @@
+## 2024-03-13
+
+| crate | version |
+| - | - |
+| foyer-storage-bench | 0.5.1 |
+
+<details>
+
+### Changes
+
+- Fix `foyer-storage-bench` build with `trace` feature.
+
+</details>
+
 ## 2024-03-12
 
 | crate | version |
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 6a4e9331..83bdbf01 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage-bench"
-version = "0.5.0"
+version = "0.5.1"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine bench tool for foyer - the hybrid cache for Rust"

From 4db04a2d9f43b1f52c7d2e0dd2d8a4046e9aa84a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 14 Mar 2024 10:55:31 +0800
Subject: [PATCH 232/261] refactor: impl Clone for config for simplification
 (#293)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/eviction/lru.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index 2608eafe..701bc5fc 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -26,7 +26,7 @@ use crate::{
     CacheContext, Key, Value,
 };
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct LruConfig {
     /// The ratio of the high priority pool occupied.
     ///

From fabed44869012992a4a9436dfa478b8bdd3361e2 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 14 Mar 2024 11:02:42 +0800
Subject: [PATCH 233/261] chore: bump foyer-memory to 0.1.1 (#294)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md            | 14 ++++++++++++++
 foyer-memory/Cargo.toml |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c88f014..42893f24 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,17 @@
+## 2024-03-14
+
+| crate | version |
+| - | - |
+| foyer-memory | 0.1.1 |
+
+<details>
+
+### Changes
+
+- Make eviction config clonable.
+
+</details>
+
 ## 2024-03-13
 
 | crate | version |
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index a2561e5f..783cc8e3 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"

From 050a3d0063e10964ed26bf4b00cf18cea6b042bf Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 21 Mar 2024 12:03:54 +0800
Subject: [PATCH 234/261] refactor: export DefaultCacheEventListener (#296)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/prelude.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index a909f2b0..75a9be35 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -16,6 +16,6 @@ pub use crate::{
     cache::{Cache, CacheEntry, Entry, EntryState, FifoCacheConfig, LfuCacheConfig, LruCacheConfig},
     context::CacheContext,
     eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig},
-    listener::CacheEventListener,
+    listener::{CacheEventListener, DefaultCacheEventListener},
     metrics::Metrics,
 };

From 524b64576941e80c451a40e0b8d1ce13922186b9 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 21 Mar 2024 12:09:56 +0800
Subject: [PATCH 235/261] chore: bump foyer-memory version (#297)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 783cc8e3..4e314d44 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"

From e76a499f7880950febaf69d22d26f6109446b41b Mon Sep 17 00:00:00 2001
From: JinYan Su <751080330@qq.com>
Date: Sun, 31 Mar 2024 14:54:24 +0800
Subject: [PATCH 236/261] feat: add eviction alg hit ratio bench (#299)

* feat: add eviction alg hit ratio bench

* refactor: use enum cache instead of generic cache

The enum cache is more easy to use since it only has two generic parameters. The generic cache is more like an internal interface.

Signed-off-by: xiaguan <751080330@qq.com>

* chore : sort foyer_memory's dependencies

Signed-off-by: xiaguan <751080330@qq.com>

* chore: use hakari to gen better dependencies

Signed-off-by: xiaguan <751080330@qq.com>

---------

Signed-off-by: xiaguan <751080330@qq.com>
---
 foyer-intrusive/src/lib.rs              |   1 +
 foyer-memory/Cargo.toml                 |   7 +-
 foyer-memory/benches/bench_hit_ratio.rs | 201 ++++++++++++++++++++++++
 foyer-memory/src/lib.rs                 |   1 +
 foyer-storage/src/lib.rs                |   1 +
 foyer-workspace-hack/Cargo.toml         |   1 +
 6 files changed, 211 insertions(+), 1 deletion(-)
 create mode 100644 foyer-memory/benches/bench_hit_ratio.rs

diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 2393da1b..74edc908 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -16,6 +16,7 @@
 #![feature(ptr_metadata)]
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
+#![feature(offset_of)]
 #![expect(clippy::new_without_default)]
 
 pub use memoffset::offset_of;
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 4e314d44..de144cba 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -25,14 +25,19 @@ itertools = "0.12"
 libc = "0.2"
 parking_lot = "0.12"
 tokio = { workspace = true }
-
 [dev-dependencies]
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 hdrhistogram = "7"
+moka = { version = "0", features = ["sync"] }
 rand = "0.8"
 rand_mt = "4.2.1"
 tempfile = "3"
+zipf = "7.0.1"
 
 [features]
 deadlock = ["parking_lot/deadlock_detection"]
+
+[[bench]]
+name = "bench_hit_ratio"
+harness = false
diff --git a/foyer-memory/benches/bench_hit_ratio.rs b/foyer-memory/benches/bench_hit_ratio.rs
new file mode 100644
index 00000000..61ca59e3
--- /dev/null
+++ b/foyer-memory/benches/bench_hit_ratio.rs
@@ -0,0 +1,201 @@
+//  Copyright 2024 MrCroxx
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::Arc;
+
+use ahash::RandomState;
+use foyer_memory::{
+    Cache, DefaultCacheEventListener, FifoCacheConfig, FifoConfig, LfuCacheConfig, LfuConfig, LruCacheConfig, LruConfig,
+};
+use rand::{distributions::Distribution, thread_rng};
+
+type CacheKey = String;
+type CacheValue = ();
+
+const ITEMS: usize = 10_000;
+const ITERATIONS: usize = 5_000_000;
+
+const SHARDS: usize = 1;
+const OBJECT_POOL_CAPACITY: usize = 16;
+/*
+inspired by pingora/tinyufo/benches/bench_hit_ratio.rs
+cargo bench --bench bench_hit_ratio
+
+zif_exp, cache_size             fifo            lru             lfu             moka
+0.90, 0.005                     16.26%          19.22%          32.38%          33.46%
+0.90, 0.01                      22.55%          26.21%          38.55%          37.93%
+0.90, 0.05                      41.10%          45.60%          55.45%          55.26%
+0.90,  0.1                      51.11%          55.72%          63.83%          64.21%
+0.90, 0.25                      66.81%          71.17%          76.21%          77.14%
+1.00, 0.005                     26.64%          31.08%          44.14%          45.61%
+1.00, 0.01                      34.37%          39.13%          50.60%          50.67%
+1.00, 0.05                      54.06%          58.79%          66.79%          66.99%
+1.00,  0.1                      63.12%          67.58%          73.92%          74.38%
+1.00, 0.25                      76.14%          79.92%          83.61%          84.33%
+1.05, 0.005                     32.63%          37.68%          50.24%          51.77%
+1.05, 0.01                      40.95%          46.09%          56.75%          57.15%
+1.05, 0.05                      60.47%          65.06%          72.07%          72.36%
+1.05,  0.1                      68.96%          73.15%          78.52%          78.96%
+1.05, 0.25                      80.42%          83.76%          86.79%          87.42%
+1.10, 0.005                     39.02%          44.52%          56.29%          57.90%
+1.10, 0.01                      47.66%          52.99%          62.64%          63.23%
+1.10, 0.05                      66.60%          70.95%          76.94%          77.25%
+1.10,  0.1                      74.26%          78.09%          82.56%          82.93%
+1.10, 0.25                      84.15%          87.06%          89.54%          90.05%
+1.50, 0.005                     81.19%          85.28%          88.91%          89.94%
+1.50, 0.01                      86.91%          89.87%          92.24%          92.78%
+1.50, 0.05                      94.75%          96.04%          96.95%          97.07%
+1.50,  0.1                      96.65%          97.51%          98.06%          98.15%
+1.50, 0.25                      98.35%          98.81%          99.04%          99.09%
+*/
+fn cache_hit(cache: Arc<Cache<CacheKey, CacheValue>>, keys: Arc<Vec<CacheKey>>) -> f64 {
+    let mut hit = 0;
+    for key in keys.iter() {
+        let value = cache.get(key);
+        if value.is_some() {
+            hit += 1;
+        } else {
+            cache.insert(key.clone(), (), 1);
+        }
+    }
+    hit as f64 / ITERATIONS as f64
+}
+
+fn moka_cache_hit(cache: &moka::sync::Cache<CacheKey, CacheValue>, keys: &[String]) -> f64 {
+    let mut hit = 0;
+    for key in keys.iter() {
+        let value = cache.get(key);
+        if value.is_some() {
+            hit += 1;
+        } else {
+            cache.insert(key.clone(), ());
+        }
+    }
+    hit as f64 / ITERATIONS as f64
+}
+
+fn new_fifo_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = FifoCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: FifoConfig {},
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::fifo(config))
+}
+
+fn new_lru_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = LruCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: LruConfig {
+            high_priority_pool_ratio: 0.1,
+        },
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::lru(config))
+}
+
+fn new_lfu_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = LfuCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: LfuConfig {
+            window_capacity_ratio: 0.1,
+            protected_capacity_ratio: 0.8,
+            cmsketch_eps: 0.001,
+            cmsketch_confidence: 0.9,
+        },
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::lfu(config))
+}
+
+fn bench_one(zif_exp: f64, cache_size_percent: f64) {
+    print!("{zif_exp:.2}, {cache_size_percent:4}\t\t\t");
+    let mut rng = thread_rng();
+    let zipf = zipf::ZipfDistribution::new(ITEMS, zif_exp).unwrap();
+
+    let cache_size = (ITEMS as f64 * cache_size_percent) as usize;
+
+    let fifo_cache = new_fifo_cache(cache_size);
+    let lru_cache = new_lru_cache(cache_size);
+    let lfu_cache = new_lfu_cache(cache_size);
+    let moka_cache = moka::sync::Cache::new(cache_size as u64);
+
+    let mut keys = Vec::with_capacity(ITERATIONS);
+    for _ in 0..ITERATIONS {
+        let key = zipf.sample(&mut rng).to_string();
+        keys.push(key.clone());
+    }
+
+    let keys = Arc::new(keys);
+
+    // Use multiple threads to simulate concurrent read-through requests.
+    let fifo_cache_hit_handle = std::thread::spawn({
+        let cache = fifo_cache.clone();
+        let keys = keys.clone();
+        move || cache_hit(cache, keys)
+    });
+
+    let lru_cache_hit_handle = std::thread::spawn({
+        let cache = lru_cache.clone();
+        let keys = keys.clone();
+        move || cache_hit(cache, keys)
+    });
+
+    let lfu_cache_hit_handle = std::thread::spawn({
+        let cache = lfu_cache.clone();
+        let keys = keys.clone();
+        move || cache_hit(cache, keys)
+    });
+
+    let moka_cache_hit_handle = std::thread::spawn({
+        let cache = moka_cache.clone();
+        let keys = keys.clone();
+        move || moka_cache_hit(&cache, &keys)
+    });
+
+    let fifo_hit_ratio = fifo_cache_hit_handle.join().unwrap();
+    let lru_hit_ratio = lru_cache_hit_handle.join().unwrap();
+    let lfu_hit_ratio = lfu_cache_hit_handle.join().unwrap();
+    let moka_hit_ratio = moka_cache_hit_handle.join().unwrap();
+
+    print!("{:.2}%\t\t", fifo_hit_ratio * 100.0);
+    print!("{:.2}%\t\t", lru_hit_ratio * 100.0);
+    print!("{:.2}%\t\t", lfu_hit_ratio * 100.0);
+    println!("{:.2}%", moka_hit_ratio * 100.0);
+}
+
+fn bench_zipf_hit() {
+    println!("zif_exp, cache_size\t\tfifo\t\tlru\t\tlfu\t\tmoka");
+    for zif_exp in [0.9, 1.0, 1.05, 1.1, 1.5] {
+        for cache_capacity in [0.005, 0.01, 0.05, 0.1, 0.25] {
+            bench_one(zif_exp, cache_capacity);
+        }
+    }
+}
+
+fn main() {
+    bench_zipf_hit();
+}
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index c4397451..8cabd2fb 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -66,6 +66,7 @@
 //! destroyed.
 
 #![feature(trait_alias)]
+#![feature(offset_of)]
 
 pub trait Key: Send + Sync + 'static + std::hash::Hash + Eq + Ord {}
 pub trait Value: Send + Sync + 'static {}
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 1578f5d8..45f87e5c 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -23,6 +23,7 @@
 #![feature(associated_type_defaults)]
 #![feature(box_into_inner)]
 #![feature(try_trait_v2)]
+#![feature(offset_of)]
 
 pub mod admission;
 pub mod buffer;
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 1cc8e191..43e9182e 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -19,6 +19,7 @@ publish = true
 [dependencies]
 ahash = { version = "0.8" }
 crossbeam-channel = { version = "0.5" }
+crossbeam-epoch = { version = "0.9" }
 crossbeam-utils = { version = "0.8" }
 either = { version = "1", default-features = false, features = ["use_std"] }
 futures-channel = { version = "0.3", features = ["sink"] }

From 6bea810c33ea09733ae6474c1eea7130f3123b46 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 31 Mar 2024 17:26:04 +0800
Subject: [PATCH 237/261] chore: update license (#295)

* chore: update license

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix license checker

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update license header

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .licenserc.yaml                                       | 2 +-
 LICENSE                                               | 2 +-
 foyer-common/src/async_queue.rs                       | 2 +-
 foyer-common/src/batch.rs                             | 2 +-
 foyer-common/src/bits.rs                              | 2 +-
 foyer-common/src/code.rs                              | 2 +-
 foyer-common/src/continuum.rs                         | 2 +-
 foyer-common/src/erwlock.rs                           | 2 +-
 foyer-common/src/lib.rs                               | 2 +-
 foyer-common/src/range.rs                             | 2 +-
 foyer-common/src/rate.rs                              | 2 +-
 foyer-common/src/rated_ticket.rs                      | 2 +-
 foyer-common/src/runtime.rs                           | 2 +-
 foyer-experimental-bench/benches/wal-bench.rs         | 2 +-
 foyer-experimental-bench/src/analyze.rs               | 2 +-
 foyer-experimental-bench/src/export.rs                | 2 +-
 foyer-experimental-bench/src/lib.rs                   | 2 +-
 foyer-experimental-bench/src/rate.rs                  | 2 +-
 foyer-experimental-bench/src/text.rs                  | 2 +-
 foyer-experimental-bench/src/utils.rs                 | 2 +-
 foyer-experimental/src/buf.rs                         | 2 +-
 foyer-experimental/src/error.rs                       | 2 +-
 foyer-experimental/src/lib.rs                         | 2 +-
 foyer-experimental/src/metrics.rs                     | 2 +-
 foyer-experimental/src/notify.rs                      | 2 +-
 foyer-experimental/src/wal.rs                         | 2 +-
 foyer-intrusive/src/collections/dlist.rs              | 2 +-
 foyer-intrusive/src/collections/duplicated_hashmap.rs | 2 +-
 foyer-intrusive/src/collections/hashmap.rs            | 2 +-
 foyer-intrusive/src/collections/mod.rs                | 2 +-
 foyer-intrusive/src/core/adapter.rs                   | 2 +-
 foyer-intrusive/src/core/mod.rs                       | 2 +-
 foyer-intrusive/src/core/pointer.rs                   | 2 +-
 foyer-intrusive/src/eviction/fifo.rs                  | 2 +-
 foyer-intrusive/src/eviction/lfu.rs                   | 2 +-
 foyer-intrusive/src/eviction/lru.rs                   | 2 +-
 foyer-intrusive/src/eviction/mod.rs                   | 2 +-
 foyer-intrusive/src/eviction/sfifo.rs                 | 2 +-
 foyer-intrusive/src/lib.rs                            | 2 +-
 foyer-memory/benches/bench_hit_ratio.rs               | 2 +-
 foyer-memory/src/cache.rs                             | 2 +-
 foyer-memory/src/context.rs                           | 2 +-
 foyer-memory/src/eviction/fifo.rs                     | 2 +-
 foyer-memory/src/eviction/lfu.rs                      | 2 +-
 foyer-memory/src/eviction/lru.rs                      | 2 +-
 foyer-memory/src/eviction/mod.rs                      | 2 +-
 foyer-memory/src/eviction/test_utils.rs               | 2 +-
 foyer-memory/src/generic.rs                           | 2 +-
 foyer-memory/src/handle.rs                            | 2 +-
 foyer-memory/src/indexer.rs                           | 2 +-
 foyer-memory/src/lib.rs                               | 2 +-
 foyer-memory/src/listener.rs                          | 2 +-
 foyer-memory/src/metrics.rs                           | 2 +-
 foyer-memory/src/prelude.rs                           | 2 +-
 foyer-storage-bench/src/analyze.rs                    | 2 +-
 foyer-storage-bench/src/export.rs                     | 2 +-
 foyer-storage-bench/src/main.rs                       | 2 +-
 foyer-storage-bench/src/rate.rs                       | 2 +-
 foyer-storage-bench/src/text.rs                       | 2 +-
 foyer-storage-bench/src/utils.rs                      | 2 +-
 foyer-storage/src/admission/mod.rs                    | 2 +-
 foyer-storage/src/admission/rated_ticket.rs           | 2 +-
 foyer-storage/src/buffer.rs                           | 2 +-
 foyer-storage/src/catalog.rs                          | 2 +-
 foyer-storage/src/compress.rs                         | 2 +-
 foyer-storage/src/device/allocator.rs                 | 2 +-
 foyer-storage/src/device/error.rs                     | 2 +-
 foyer-storage/src/device/fs.rs                        | 2 +-
 foyer-storage/src/device/mod.rs                       | 2 +-
 foyer-storage/src/error.rs                            | 2 +-
 foyer-storage/src/flusher.rs                          | 2 +-
 foyer-storage/src/generic.rs                          | 2 +-
 foyer-storage/src/indices.rs                          | 2 +-
 foyer-storage/src/judge.rs                            | 2 +-
 foyer-storage/src/lazy.rs                             | 2 +-
 foyer-storage/src/lib.rs                              | 2 +-
 foyer-storage/src/metrics.rs                          | 2 +-
 foyer-storage/src/reclaimer.rs                        | 2 +-
 foyer-storage/src/region.rs                           | 2 +-
 foyer-storage/src/region_manager.rs                   | 2 +-
 foyer-storage/src/reinsertion/exist.rs                | 2 +-
 foyer-storage/src/reinsertion/mod.rs                  | 2 +-
 foyer-storage/src/reinsertion/rated_ticket.rs         | 2 +-
 foyer-storage/src/runtime.rs                          | 2 +-
 foyer-storage/src/storage.rs                          | 2 +-
 foyer-storage/src/store.rs                            | 2 +-
 foyer-storage/src/test_utils.rs                       | 2 +-
 foyer-storage/tests/storage_test.rs                   | 2 +-
 foyer-workspace-hack/build.rs                         | 2 +-
 foyer-workspace-hack/src/lib.rs                       | 2 +-
 foyer/src/lib.rs                                      | 2 +-
 91 files changed, 91 insertions(+), 91 deletions(-)

diff --git a/.licenserc.yaml b/.licenserc.yaml
index 33c44e0d..bfb4ceb2 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -1,7 +1,7 @@
 header:
   license:
     spdx-id: Apache-2.0
-    copyright-owner: MrCroxx
+    copyright-owner: Foyer Project Authors
 
   paths:
     - "**/*.rs"
diff --git a/LICENSE b/LICENSE
index f5ef119f..7e01351a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 MrCroxx
+   Copyright 2024 Foyer Project Authors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/async_queue.rs b/foyer-common/src/async_queue.rs
index 6c9017bd..fbaba58f 100644
--- a/foyer-common/src/async_queue.rs
+++ b/foyer-common/src/async_queue.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/batch.rs b/foyer-common/src/batch.rs
index b4300a49..a9a72ea6 100644
--- a/foyer-common/src/batch.rs
+++ b/foyer-common/src/batch.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/bits.rs b/foyer-common/src/bits.rs
index 378b3747..c7bc8aee 100644
--- a/foyer-common/src/bits.rs
+++ b/foyer-common/src/bits.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 7c2cb8ac..6e127119 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/continuum.rs b/foyer-common/src/continuum.rs
index a9fa147c..4efac1d4 100644
--- a/foyer-common/src/continuum.rs
+++ b/foyer-common/src/continuum.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/erwlock.rs b/foyer-common/src/erwlock.rs
index bc820901..9bd48f6f 100644
--- a/foyer-common/src/erwlock.rs
+++ b/foyer-common/src/erwlock.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 7d570a09..46e94ee4 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/range.rs b/foyer-common/src/range.rs
index c986c827..bb107413 100644
--- a/foyer-common/src/range.rs
+++ b/foyer-common/src/range.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/rate.rs b/foyer-common/src/rate.rs
index 46d6105f..884fbc4a 100644
--- a/foyer-common/src/rate.rs
+++ b/foyer-common/src/rate.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/rated_ticket.rs b/foyer-common/src/rated_ticket.rs
index 078ccaf4..95d2a235 100644
--- a/foyer-common/src/rated_ticket.rs
+++ b/foyer-common/src/rated_ticket.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index e97581e1..703171b2 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental-bench/benches/wal-bench.rs b/foyer-experimental-bench/benches/wal-bench.rs
index 1eed2650..4af9b033 100644
--- a/foyer-experimental-bench/benches/wal-bench.rs
+++ b/foyer-experimental-bench/benches/wal-bench.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental-bench/src/analyze.rs b/foyer-experimental-bench/src/analyze.rs
index 05db6137..4c50cfa5 100644
--- a/foyer-experimental-bench/src/analyze.rs
+++ b/foyer-experimental-bench/src/analyze.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental-bench/src/export.rs b/foyer-experimental-bench/src/export.rs
index 0659d979..884aeb46 100644
--- a/foyer-experimental-bench/src/export.rs
+++ b/foyer-experimental-bench/src/export.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental-bench/src/lib.rs b/foyer-experimental-bench/src/lib.rs
index cbaf3f27..1509de6c 100644
--- a/foyer-experimental-bench/src/lib.rs
+++ b/foyer-experimental-bench/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental-bench/src/rate.rs b/foyer-experimental-bench/src/rate.rs
index 5be56a63..e7d86a34 100644
--- a/foyer-experimental-bench/src/rate.rs
+++ b/foyer-experimental-bench/src/rate.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental-bench/src/text.rs b/foyer-experimental-bench/src/text.rs
index 614d4365..2ec8f8f2 100644
--- a/foyer-experimental-bench/src/text.rs
+++ b/foyer-experimental-bench/src/text.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental-bench/src/utils.rs b/foyer-experimental-bench/src/utils.rs
index 84ad7ce9..f640b3c9 100644
--- a/foyer-experimental-bench/src/utils.rs
+++ b/foyer-experimental-bench/src/utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental/src/buf.rs b/foyer-experimental/src/buf.rs
index 28270cc8..2366bf1f 100644
--- a/foyer-experimental/src/buf.rs
+++ b/foyer-experimental/src/buf.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental/src/error.rs b/foyer-experimental/src/error.rs
index 61500387..cb50531c 100644
--- a/foyer-experimental/src/error.rs
+++ b/foyer-experimental/src/error.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental/src/lib.rs b/foyer-experimental/src/lib.rs
index 3233ff9f..1d8ca5e2 100644
--- a/foyer-experimental/src/lib.rs
+++ b/foyer-experimental/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental/src/metrics.rs b/foyer-experimental/src/metrics.rs
index 2d1d9920..5b07df73 100644
--- a/foyer-experimental/src/metrics.rs
+++ b/foyer-experimental/src/metrics.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental/src/notify.rs b/foyer-experimental/src/notify.rs
index 5c0b589a..e98a409e 100644
--- a/foyer-experimental/src/notify.rs
+++ b/foyer-experimental/src/notify.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-experimental/src/wal.rs b/foyer-experimental/src/wal.rs
index ba33ed2a..517e63b6 100644
--- a/foyer-experimental/src/wal.rs
+++ b/foyer-experimental/src/wal.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/collections/dlist.rs b/foyer-intrusive/src/collections/dlist.rs
index de3ccb7d..2712808d 100644
--- a/foyer-intrusive/src/collections/dlist.rs
+++ b/foyer-intrusive/src/collections/dlist.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/collections/duplicated_hashmap.rs b/foyer-intrusive/src/collections/duplicated_hashmap.rs
index e0a541fa..7b3da1e7 100644
--- a/foyer-intrusive/src/collections/duplicated_hashmap.rs
+++ b/foyer-intrusive/src/collections/duplicated_hashmap.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/collections/hashmap.rs b/foyer-intrusive/src/collections/hashmap.rs
index 0c5091b6..ff251070 100644
--- a/foyer-intrusive/src/collections/hashmap.rs
+++ b/foyer-intrusive/src/collections/hashmap.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/collections/mod.rs b/foyer-intrusive/src/collections/mod.rs
index d3b15ac6..a43fff09 100644
--- a/foyer-intrusive/src/collections/mod.rs
+++ b/foyer-intrusive/src/collections/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index f8a4946a..eb09afbd 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/core/mod.rs b/foyer-intrusive/src/core/mod.rs
index f79f4215..98f30086 100644
--- a/foyer-intrusive/src/core/mod.rs
+++ b/foyer-intrusive/src/core/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/core/pointer.rs b/foyer-intrusive/src/core/pointer.rs
index 8c74d48d..7bac29f2 100644
--- a/foyer-intrusive/src/core/pointer.rs
+++ b/foyer-intrusive/src/core/pointer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/fifo.rs b/foyer-intrusive/src/eviction/fifo.rs
index b3615299..50810894 100644
--- a/foyer-intrusive/src/eviction/fifo.rs
+++ b/foyer-intrusive/src/eviction/fifo.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index 215e7cfd..cd1ae9c1 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/lru.rs b/foyer-intrusive/src/eviction/lru.rs
index 25ccdaf8..752cb078 100644
--- a/foyer-intrusive/src/eviction/lru.rs
+++ b/foyer-intrusive/src/eviction/lru.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index 6055040e..614e1db6 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/eviction/sfifo.rs b/foyer-intrusive/src/eviction/sfifo.rs
index c0db363f..483c882b 100644
--- a/foyer-intrusive/src/eviction/sfifo.rs
+++ b/foyer-intrusive/src/eviction/sfifo.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 74edc908..7ea663d5 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/benches/bench_hit_ratio.rs b/foyer-memory/benches/bench_hit_ratio.rs
index 61ca59e3..a78daad4 100644
--- a/foyer-memory/benches/bench_hit_ratio.rs
+++ b/foyer-memory/benches/bench_hit_ratio.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 7f3786d0..5ca929aa 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/context.rs b/foyer-memory/src/context.rs
index 3157a6f2..3ca02c9d 100644
--- a/foyer-memory/src/context.rs
+++ b/foyer-memory/src/context.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index c611fdef..c42e27cb 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index e6dc8d3f..e885a007 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index 701bc5fc..3cfc44d7 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index 57138cae..d18bd18b 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/test_utils.rs b/foyer-memory/src/eviction/test_utils.rs
index e266d405..39b57347 100644
--- a/foyer-memory/src/eviction/test_utils.rs
+++ b/foyer-memory/src/eviction/test_utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index 328607b8..6352c368 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
index 46f33176..602b4bf7 100644
--- a/foyer-memory/src/handle.rs
+++ b/foyer-memory/src/handle.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/indexer.rs b/foyer-memory/src/indexer.rs
index df9fd630..b22cde15 100644
--- a/foyer-memory/src/indexer.rs
+++ b/foyer-memory/src/indexer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 8cabd2fb..cc504e32 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/listener.rs b/foyer-memory/src/listener.rs
index a858c306..6099bfb3 100644
--- a/foyer-memory/src/listener.rs
+++ b/foyer-memory/src/listener.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/metrics.rs b/foyer-memory/src/metrics.rs
index a5bcbf3c..4644730b 100644
--- a/foyer-memory/src/metrics.rs
+++ b/foyer-memory/src/metrics.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index 75a9be35..a789b8d1 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/analyze.rs b/foyer-storage-bench/src/analyze.rs
index 05db6137..4c50cfa5 100644
--- a/foyer-storage-bench/src/analyze.rs
+++ b/foyer-storage-bench/src/analyze.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/export.rs b/foyer-storage-bench/src/export.rs
index 0659d979..884aeb46 100644
--- a/foyer-storage-bench/src/export.rs
+++ b/foyer-storage-bench/src/export.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index d95fb6ea..c2efeb34 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/rate.rs b/foyer-storage-bench/src/rate.rs
index 5be56a63..e7d86a34 100644
--- a/foyer-storage-bench/src/rate.rs
+++ b/foyer-storage-bench/src/rate.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/text.rs b/foyer-storage-bench/src/text.rs
index 614d4365..2ec8f8f2 100644
--- a/foyer-storage-bench/src/text.rs
+++ b/foyer-storage-bench/src/text.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
index 4e5a6a37..c2a40a0f 100644
--- a/foyer-storage-bench/src/utils.rs
+++ b/foyer-storage-bench/src/utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index 670f27b2..dcff9e98 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/admission/rated_ticket.rs b/foyer-storage/src/admission/rated_ticket.rs
index 2a9c8884..c2877c34 100644
--- a/foyer-storage/src/admission/rated_ticket.rs
+++ b/foyer-storage/src/admission/rated_ticket.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index efdf04a6..dfc0c179 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 0c0d9ca6..3177995b 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/compress.rs b/foyer-storage/src/compress.rs
index 20020ca5..f87c1e90 100644
--- a/foyer-storage/src/compress.rs
+++ b/foyer-storage/src/compress.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/allocator.rs b/foyer-storage/src/device/allocator.rs
index 8c43ab7a..c848417b 100644
--- a/foyer-storage/src/device/allocator.rs
+++ b/foyer-storage/src/device/allocator.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/error.rs b/foyer-storage/src/device/error.rs
index e6708507..e9aee540 100644
--- a/foyer-storage/src/device/error.rs
+++ b/foyer-storage/src/device/error.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index e7ddec56..d7ba50d2 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 5cc71905..fc52bc38 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index ea7d055c..55c9e254 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 102c30f0..6b19a329 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index cd27bbea..490c3915 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/indices.rs b/foyer-storage/src/indices.rs
index 1cd2cf5b..ae20d276 100644
--- a/foyer-storage/src/indices.rs
+++ b/foyer-storage/src/indices.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/judge.rs b/foyer-storage/src/judge.rs
index 695085c4..b347cacd 100644
--- a/foyer-storage/src/judge.rs
+++ b/foyer-storage/src/judge.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/lazy.rs b/foyer-storage/src/lazy.rs
index 19e85c9e..a93f8542 100644
--- a/foyer-storage/src/lazy.rs
+++ b/foyer-storage/src/lazy.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 45f87e5c..d9e98496 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index ead1232f..2512fc2c 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reclaimer.rs b/foyer-storage/src/reclaimer.rs
index f87d865b..864cb66a 100644
--- a/foyer-storage/src/reclaimer.rs
+++ b/foyer-storage/src/reclaimer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index bc565b10..94a7dc37 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/region_manager.rs b/foyer-storage/src/region_manager.rs
index bde487bf..393b1493 100644
--- a/foyer-storage/src/region_manager.rs
+++ b/foyer-storage/src/region_manager.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reinsertion/exist.rs b/foyer-storage/src/reinsertion/exist.rs
index 2e10d50f..d1764ddb 100644
--- a/foyer-storage/src/reinsertion/exist.rs
+++ b/foyer-storage/src/reinsertion/exist.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index e39b08a0..493c941f 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/reinsertion/rated_ticket.rs b/foyer-storage/src/reinsertion/rated_ticket.rs
index 2e313faa..ccc39027 100644
--- a/foyer-storage/src/reinsertion/rated_ticket.rs
+++ b/foyer-storage/src/reinsertion/rated_ticket.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index 8a7f5a3f..6212172e 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index ad3401f0..3909731a 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 625ca772..0d5b8f5f 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index 1ca8541a..25fffc85 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 640171e1..97bc7143 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-workspace-hack/build.rs b/foyer-workspace-hack/build.rs
index 540913fa..91682caf 100644
--- a/foyer-workspace-hack/build.rs
+++ b/foyer-workspace-hack/build.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-workspace-hack/src/lib.rs b/foyer-workspace-hack/src/lib.rs
index fffa08f9..2b5d0f4b 100644
--- a/foyer-workspace-hack/src/lib.rs
+++ b/foyer-workspace-hack/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 6a90482b..191d0932 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 MrCroxx
+//  Copyright 2024 Foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.

From 41eebba5c0166b82145b70113f8041d5d967f5c7 Mon Sep 17 00:00:00 2001
From: Liquan Pei <liquanpei@gmail.com>
Date: Mon, 8 Apr 2024 00:40:18 -0700
Subject: [PATCH 238/261] feat: add S3Fifo eviction for memory (#303)

* Add S3Fifo eviction for memory

* fix: refine s3fifo, fix some bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix license

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: expose s3fifo, fix hakari, add s3fifo fuzzy test

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* bench: add s3fifo to hit ratio bench

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add s3fifo uts

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
Co-authored-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-intrusive/src/core/adapter.rs     |   6 +
 foyer-intrusive/src/eviction/lfu.rs     |   2 +-
 foyer-intrusive/src/lib.rs              |   2 +
 foyer-memory/Cargo.toml                 |   1 +
 foyer-memory/benches/bench_hit_ratio.rs |  81 +++--
 foyer-memory/src/cache.rs               |  80 ++++-
 foyer-memory/src/eviction/mod.rs        |   1 +
 foyer-memory/src/eviction/s3fifo.rs     | 420 ++++++++++++++++++++++++
 foyer-memory/src/prelude.rs             |   4 +-
 foyer-workspace-hack/Cargo.toml         |   2 +
 10 files changed, 567 insertions(+), 32 deletions(-)
 create mode 100644 foyer-memory/src/eviction/s3fifo.rs

diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index eb09afbd..3f98d74e 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -117,6 +117,8 @@ pub unsafe trait PriorityAdapter: Adapter {
 /// # Examples
 ///
 /// ```
+/// #![feature(offset_of)]
+///
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
 /// use foyer_intrusive::core::pointer::Pointer;
@@ -213,6 +215,8 @@ macro_rules! intrusive_adapter {
 /// # Examples
 ///
 /// ```
+/// #![feature(offset_of)]
+///
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
 /// use foyer_intrusive::core::pointer::Pointer;
@@ -282,6 +286,8 @@ macro_rules! key_adapter {
 /// # Examples
 ///
 /// ```
+/// #![feature(offset_of)]
+///
 /// use foyer_intrusive::{intrusive_adapter, priority_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, PriorityAdapter, Link};
 /// use foyer_intrusive::core::pointer::Pointer;
diff --git a/foyer-intrusive/src/eviction/lfu.rs b/foyer-intrusive/src/eviction/lfu.rs
index cd1ae9c1..dc62da52 100644
--- a/foyer-intrusive/src/eviction/lfu.rs
+++ b/foyer-intrusive/src/eviction/lfu.rs
@@ -468,7 +468,7 @@ where
                     link
                 }
                 (Some(link_main), Some(link_tiny)) => {
-                    // Eviction from tiny or main depending on whether the tiny handle woould be
+                    // Eviction from tiny or main depending on whether the tiny handle would be
                     // admitted to main cachce. If it would be, evict from main cache, otherwise
                     // from tiny cache.
                     if self.lfu.admit_to_main(link_main.raw(), link_tiny.raw()) {
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 7ea663d5..4241d821 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -27,6 +27,8 @@ pub use memoffset::offset_of;
 /// # Examples
 ///
 /// ```
+/// #![feature(offset_of)]
+///
 /// use foyer_intrusive::container_of;
 ///
 /// struct S { x: u32, y: u32 };
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index de144cba..ffe808fd 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -25,6 +25,7 @@ itertools = "0.12"
 libc = "0.2"
 parking_lot = "0.12"
 tokio = { workspace = true }
+
 [dev-dependencies]
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
diff --git a/foyer-memory/benches/bench_hit_ratio.rs b/foyer-memory/benches/bench_hit_ratio.rs
index a78daad4..3cb51fa7 100644
--- a/foyer-memory/benches/bench_hit_ratio.rs
+++ b/foyer-memory/benches/bench_hit_ratio.rs
@@ -16,7 +16,8 @@ use std::sync::Arc;
 
 use ahash::RandomState;
 use foyer_memory::{
-    Cache, DefaultCacheEventListener, FifoCacheConfig, FifoConfig, LfuCacheConfig, LfuConfig, LruCacheConfig, LruConfig,
+    Cache, DefaultCacheEventListener, FifoCacheConfig, FifoConfig, LfuCacheConfig, LfuConfig, LruCacheConfig,
+    LruConfig, S3FifoCacheConfig, S3FifoConfig,
 };
 use rand::{distributions::Distribution, thread_rng};
 
@@ -32,32 +33,32 @@ const OBJECT_POOL_CAPACITY: usize = 16;
 inspired by pingora/tinyufo/benches/bench_hit_ratio.rs
 cargo bench --bench bench_hit_ratio
 
-zif_exp, cache_size             fifo            lru             lfu             moka
-0.90, 0.005                     16.26%          19.22%          32.38%          33.46%
-0.90, 0.01                      22.55%          26.21%          38.55%          37.93%
-0.90, 0.05                      41.10%          45.60%          55.45%          55.26%
-0.90,  0.1                      51.11%          55.72%          63.83%          64.21%
-0.90, 0.25                      66.81%          71.17%          76.21%          77.14%
-1.00, 0.005                     26.64%          31.08%          44.14%          45.61%
-1.00, 0.01                      34.37%          39.13%          50.60%          50.67%
-1.00, 0.05                      54.06%          58.79%          66.79%          66.99%
-1.00,  0.1                      63.12%          67.58%          73.92%          74.38%
-1.00, 0.25                      76.14%          79.92%          83.61%          84.33%
-1.05, 0.005                     32.63%          37.68%          50.24%          51.77%
-1.05, 0.01                      40.95%          46.09%          56.75%          57.15%
-1.05, 0.05                      60.47%          65.06%          72.07%          72.36%
-1.05,  0.1                      68.96%          73.15%          78.52%          78.96%
-1.05, 0.25                      80.42%          83.76%          86.79%          87.42%
-1.10, 0.005                     39.02%          44.52%          56.29%          57.90%
-1.10, 0.01                      47.66%          52.99%          62.64%          63.23%
-1.10, 0.05                      66.60%          70.95%          76.94%          77.25%
-1.10,  0.1                      74.26%          78.09%          82.56%          82.93%
-1.10, 0.25                      84.15%          87.06%          89.54%          90.05%
-1.50, 0.005                     81.19%          85.28%          88.91%          89.94%
-1.50, 0.01                      86.91%          89.87%          92.24%          92.78%
-1.50, 0.05                      94.75%          96.04%          96.95%          97.07%
-1.50,  0.1                      96.65%          97.51%          98.06%          98.15%
-1.50, 0.25                      98.35%          98.81%          99.04%          99.09%
+zif_exp, cache_size             fifo            lru             lfu             s3fifo          moka
+0.90, 0.005                     16.24%          19.22%          32.37%          32.39%          33.50%
+0.90, 0.01                      22.55%          26.20%          38.54%          39.20%          37.92%
+0.90, 0.05                      41.05%          45.56%          55.37%          56.63%          55.25%
+0.90,  0.1                      51.06%          55.68%          63.82%          65.06%          64.20%
+0.90, 0.25                      66.81%          71.17%          76.21%          77.26%          77.12%
+1.00, 0.005                     26.62%          31.10%          44.16%          44.15%          45.62%
+1.00, 0.01                      34.38%          39.17%          50.63%          51.29%          50.72%
+1.00, 0.05                      54.04%          58.76%          66.79%          67.85%          66.89%
+1.00,  0.1                      63.15%          67.60%          73.93%          74.92%          74.38%
+1.00, 0.25                      76.18%          79.95%          83.63%          84.39%          84.38%
+1.05, 0.005                     32.67%          37.71%          50.26%          50.21%          51.85%
+1.05, 0.01                      40.97%          46.10%          56.74%          57.40%          57.09%
+1.05, 0.05                      60.44%          65.03%          72.04%          73.02%          72.28%
+1.05,  0.1                      68.93%          73.12%          78.49%          79.37%          79.00%
+1.05, 0.25                      80.38%          83.73%          86.78%          87.42%          87.41%
+1.10, 0.005                     39.02%          44.50%          56.26%          56.20%          57.90%
+1.10, 0.01                      47.60%          52.93%          62.61%          63.24%          63.05%
+1.10, 0.05                      66.59%          70.95%          76.92%          77.76%          77.27%
+1.10,  0.1                      74.24%          78.07%          82.54%          83.28%          83.00%
+1.10, 0.25                      84.18%          87.10%          89.57%          90.06%          90.08%
+1.50, 0.005                     81.17%          85.27%          88.90%          89.10%          89.89%
+1.50, 0.01                      86.91%          89.87%          92.25%          92.56%          92.79%
+1.50, 0.05                      94.77%          96.04%          96.96%          97.10%          97.07%
+1.50,  0.1                      96.65%          97.50%          98.06%          98.14%          98.15%
+1.50, 0.25                      98.36%          98.81%          99.04%          99.06%          99.09%
 */
 fn cache_hit(cache: Arc<Cache<CacheKey, CacheValue>>, keys: Arc<Vec<CacheKey>>) -> f64 {
     let mut hit = 0;
@@ -131,6 +132,21 @@ fn new_lfu_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
     Arc::new(Cache::lfu(config))
 }
 
+fn new_s3fifo_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = S3FifoCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: S3FifoConfig {
+            small_queue_capacity_ratio: 0.1,
+        },
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::s3fifo(config))
+}
+
 fn bench_one(zif_exp: f64, cache_size_percent: f64) {
     print!("{zif_exp:.2}, {cache_size_percent:4}\t\t\t");
     let mut rng = thread_rng();
@@ -141,6 +157,7 @@ fn bench_one(zif_exp: f64, cache_size_percent: f64) {
     let fifo_cache = new_fifo_cache(cache_size);
     let lru_cache = new_lru_cache(cache_size);
     let lfu_cache = new_lfu_cache(cache_size);
+    let s3fifo_cache = new_s3fifo_cache(cache_size);
     let moka_cache = moka::sync::Cache::new(cache_size as u64);
 
     let mut keys = Vec::with_capacity(ITERATIONS);
@@ -170,6 +187,12 @@ fn bench_one(zif_exp: f64, cache_size_percent: f64) {
         move || cache_hit(cache, keys)
     });
 
+    let s3fifo_cache_hit_handle = std::thread::spawn({
+        let cache = s3fifo_cache.clone();
+        let keys = keys.clone();
+        move || cache_hit(cache, keys)
+    });
+
     let moka_cache_hit_handle = std::thread::spawn({
         let cache = moka_cache.clone();
         let keys = keys.clone();
@@ -179,16 +202,18 @@ fn bench_one(zif_exp: f64, cache_size_percent: f64) {
     let fifo_hit_ratio = fifo_cache_hit_handle.join().unwrap();
     let lru_hit_ratio = lru_cache_hit_handle.join().unwrap();
     let lfu_hit_ratio = lfu_cache_hit_handle.join().unwrap();
+    let s3fifo_hit_ratio = s3fifo_cache_hit_handle.join().unwrap();
     let moka_hit_ratio = moka_cache_hit_handle.join().unwrap();
 
     print!("{:.2}%\t\t", fifo_hit_ratio * 100.0);
     print!("{:.2}%\t\t", lru_hit_ratio * 100.0);
     print!("{:.2}%\t\t", lfu_hit_ratio * 100.0);
+    print!("{:.2}%\t\t", s3fifo_hit_ratio * 100.0);
     println!("{:.2}%", moka_hit_ratio * 100.0);
 }
 
 fn bench_zipf_hit() {
-    println!("zif_exp, cache_size\t\tfifo\t\tlru\t\tlfu\t\tmoka");
+    println!("zif_exp, cache_size\t\tfifo\t\tlru\t\tlfu\t\ts3fifo\t\tmoka");
     for zif_exp in [0.9, 1.0, 1.05, 1.1, 1.5] {
         for cache_capacity in [0.005, 0.01, 0.05, 0.1, 0.25] {
             bench_one(zif_exp, cache_capacity);
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 5ca929aa..d1a59d69 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -24,6 +24,7 @@ use crate::{
         fifo::{Fifo, FifoHandle},
         lfu::{Lfu, LfuHandle},
         lru::{Lru, LruHandle},
+        s3fifo::{S3Fifo, S3FifoHandle},
     },
     generic::{CacheConfig, GenericCache, GenericCacheEntry, GenericEntry},
     indexer::HashTableIndexer,
@@ -56,6 +57,15 @@ pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomStat
 pub type LfuEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     GenericEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S, ER>;
 
+pub type S3FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCache<K, V, S3FifoHandle<K, V>, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
+pub type S3FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    CacheConfig<S3Fifo<K, V>, L, S>;
+pub type S3FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericCacheEntry<K, V, S3FifoHandle<K, V>, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
+pub type S3FifoEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    GenericEntry<K, V, S3FifoHandle<K, V>, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S, ER>;
+
 pub enum CacheEntry<K, V, L, S = RandomState>
 where
     K: Key,
@@ -66,6 +76,7 @@ where
     Fifo(FifoCacheEntry<K, V, L, S>),
     Lru(LruCacheEntry<K, V, L, S>),
     Lfu(LfuCacheEntry<K, V, L, S>),
+    S3Fifo(S3FifoCacheEntry<K, V, L, S>),
 }
 
 impl<K, V, L, S> Clone for CacheEntry<K, V, L, S>
@@ -80,6 +91,7 @@ where
             Self::Fifo(entry) => Self::Fifo(entry.clone()),
             Self::Lru(entry) => Self::Lru(entry.clone()),
             Self::Lfu(entry) => Self::Lfu(entry.clone()),
+            Self::S3Fifo(entry) => Self::S3Fifo(entry.clone()),
         }
     }
 }
@@ -98,6 +110,7 @@ where
             CacheEntry::Fifo(entry) => entry.deref(),
             CacheEntry::Lru(entry) => entry.deref(),
             CacheEntry::Lfu(entry) => entry.deref(),
+            CacheEntry::S3Fifo(entry) => entry.deref(),
         }
     }
 }
@@ -138,6 +151,18 @@ where
     }
 }
 
+impl<K, V, L, S> From<S3FifoCacheEntry<K, V, L, S>> for CacheEntry<K, V, L, S>
+where
+    K: Key,
+    V: Value,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn from(entry: S3FifoCacheEntry<K, V, L, S>) -> Self {
+        Self::S3Fifo(entry)
+    }
+}
+
 impl<K, V, L, S> CacheEntry<K, V, L, S>
 where
     K: Key,
@@ -150,6 +175,7 @@ where
             CacheEntry::Fifo(entry) => entry.key(),
             CacheEntry::Lru(entry) => entry.key(),
             CacheEntry::Lfu(entry) => entry.key(),
+            CacheEntry::S3Fifo(entry) => entry.key(),
         }
     }
 
@@ -158,6 +184,7 @@ where
             CacheEntry::Fifo(entry) => entry.value(),
             CacheEntry::Lru(entry) => entry.value(),
             CacheEntry::Lfu(entry) => entry.value(),
+            CacheEntry::S3Fifo(entry) => entry.value(),
         }
     }
 
@@ -166,6 +193,7 @@ where
             CacheEntry::Fifo(entry) => entry.context().clone().into(),
             CacheEntry::Lru(entry) => entry.context().clone().into(),
             CacheEntry::Lfu(entry) => entry.context().clone().into(),
+            CacheEntry::S3Fifo(entry) => entry.context().clone().into(),
         }
     }
 
@@ -174,6 +202,7 @@ where
             CacheEntry::Fifo(entry) => entry.charge(),
             CacheEntry::Lru(entry) => entry.charge(),
             CacheEntry::Lfu(entry) => entry.charge(),
+            CacheEntry::S3Fifo(entry) => entry.charge(),
         }
     }
 
@@ -182,6 +211,7 @@ where
             CacheEntry::Fifo(entry) => entry.refs(),
             CacheEntry::Lru(entry) => entry.refs(),
             CacheEntry::Lfu(entry) => entry.refs(),
+            CacheEntry::S3Fifo(entry) => entry.refs(),
         }
     }
 }
@@ -196,6 +226,7 @@ where
     Fifo(Arc<FifoCache<K, V, L, S>>),
     Lru(Arc<LruCache<K, V, L, S>>),
     Lfu(Arc<LfuCache<K, V, L, S>>),
+    S3Fifo(Arc<S3FifoCache<K, V, L, S>>),
 }
 
 impl<K, V, L, S> Clone for Cache<K, V, L, S>
@@ -210,6 +241,7 @@ where
             Self::Fifo(cache) => Self::Fifo(cache.clone()),
             Self::Lru(cache) => Self::Lru(cache.clone()),
             Self::Lfu(cache) => Self::Lfu(cache.clone()),
+            Self::S3Fifo(cache) => Self::S3Fifo(cache.clone()),
         }
     }
 }
@@ -233,11 +265,16 @@ where
         Self::Lfu(Arc::new(GenericCache::new(config)))
     }
 
+    pub fn s3fifo(config: S3FifoCacheConfig<K, V, L, S>) -> Self {
+        Self::S3Fifo(Arc::new(GenericCache::new(config)))
+    }
+
     pub fn insert(&self, key: K, value: V, charge: usize) -> CacheEntry<K, V, L, S> {
         match self {
             Cache::Fifo(cache) => cache.insert(key, value, charge).into(),
             Cache::Lru(cache) => cache.insert(key, value, charge).into(),
             Cache::Lfu(cache) => cache.insert(key, value, charge).into(),
+            Cache::S3Fifo(cache) => cache.insert(key, value, charge).into(),
         }
     }
 
@@ -252,6 +289,7 @@ where
             Cache::Fifo(cache) => cache.insert_with_context(key, value, charge, context).into(),
             Cache::Lru(cache) => cache.insert_with_context(key, value, charge, context).into(),
             Cache::Lfu(cache) => cache.insert_with_context(key, value, charge, context).into(),
+            Cache::S3Fifo(cache) => cache.insert_with_context(key, value, charge, context).into(),
         }
     }
 
@@ -260,6 +298,7 @@ where
             Cache::Fifo(cache) => cache.remove(key),
             Cache::Lru(cache) => cache.remove(key),
             Cache::Lfu(cache) => cache.remove(key),
+            Cache::S3Fifo(cache) => cache.remove(key),
         }
     }
 
@@ -268,6 +307,7 @@ where
             Cache::Fifo(cache) => cache.get(key).map(CacheEntry::from),
             Cache::Lru(cache) => cache.get(key).map(CacheEntry::from),
             Cache::Lfu(cache) => cache.get(key).map(CacheEntry::from),
+            Cache::S3Fifo(cache) => cache.get(key).map(CacheEntry::from),
         }
     }
 
@@ -276,6 +316,7 @@ where
             Cache::Fifo(cache) => cache.clear(),
             Cache::Lru(cache) => cache.clear(),
             Cache::Lfu(cache) => cache.clear(),
+            Cache::S3Fifo(cache) => cache.clear(),
         }
     }
 
@@ -284,6 +325,7 @@ where
             Cache::Fifo(cache) => cache.capacity(),
             Cache::Lru(cache) => cache.capacity(),
             Cache::Lfu(cache) => cache.capacity(),
+            Cache::S3Fifo(cache) => cache.capacity(),
         }
     }
 
@@ -292,6 +334,7 @@ where
             Cache::Fifo(cache) => cache.usage(),
             Cache::Lru(cache) => cache.usage(),
             Cache::Lfu(cache) => cache.usage(),
+            Cache::S3Fifo(cache) => cache.usage(),
         }
     }
 
@@ -300,6 +343,7 @@ where
             Cache::Fifo(cache) => cache.metrics(),
             Cache::Lru(cache) => cache.metrics(),
             Cache::Lfu(cache) => cache.metrics(),
+            Cache::S3Fifo(cache) => cache.metrics(),
         }
     }
 }
@@ -315,6 +359,7 @@ where
     Fifo(FifoEntry<K, V, ER, L, S>),
     Lru(LruEntry<K, V, ER, L, S>),
     Lfu(LfuEntry<K, V, ER, L, S>),
+    S3Fifo(S3FifoEntry<K, V, ER, L, S>),
 }
 
 impl<K, V, ER, L, S> From<FifoEntry<K, V, ER, L, S>> for Entry<K, V, ER, L, S>
@@ -356,6 +401,19 @@ where
     }
 }
 
+impl<K, V, ER, L, S> From<S3FifoEntry<K, V, ER, L, S>> for Entry<K, V, ER, L, S>
+where
+    K: Key + Clone,
+    V: Value,
+    ER: std::error::Error,
+    L: CacheEventListener<K, V>,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    fn from(entry: S3FifoEntry<K, V, ER, L, S>) -> Self {
+        Self::S3Fifo(entry)
+    }
+}
+
 impl<K, V, ER, L, S> Future for Entry<K, V, ER, L, S>
 where
     K: Key + Clone,
@@ -371,6 +429,7 @@ where
             Entry::Fifo(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)),
             Entry::Lru(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)),
             Entry::Lfu(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)),
+            Entry::S3Fifo(entry) => entry.poll_unpin(cx).map(|res| res.map(CacheEntry::from)),
         }
     }
 }
@@ -423,6 +482,7 @@ where
             Cache::Fifo(cache) => Entry::from(cache.entry(key, f)),
             Cache::Lru(cache) => Entry::from(cache.entry(key, f)),
             Cache::Lfu(cache) => Entry::from(cache.entry(key, f)),
+            Cache::S3Fifo(cache) => Entry::from(cache.entry(key, f)),
         }
     }
 }
@@ -436,7 +496,7 @@ mod tests {
     use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng};
 
     use super::*;
-    use crate::{FifoConfig, LfuConfig, LruConfig};
+    use crate::{eviction::s3fifo::S3FifoConfig, FifoConfig, LfuConfig, LruConfig};
 
     const CAPACITY: usize = 100;
     const SHARDS: usize = 4;
@@ -485,6 +545,19 @@ mod tests {
         })
     }
 
+    fn s3fifo() -> Cache<u64, u64> {
+        Cache::s3fifo(S3FifoCacheConfig {
+            capacity: CAPACITY,
+            shards: SHARDS,
+            eviction_config: S3FifoConfig {
+                small_queue_capacity_ratio: 0.1,
+            },
+            object_pool_capacity: OBJECT_POOL_CAPACITY,
+            hash_builder: RandomState::default(),
+            event_listener: DefaultCacheEventListener::default(),
+        })
+    }
+
     fn init_cache(cache: &Cache<u64, u64>, rng: &mut StdRng) {
         let mut v = RANGE.collect_vec();
         v.shuffle(rng);
@@ -559,4 +632,9 @@ mod tests {
     async fn test_lfu_cache() {
         case(lfu()).await
     }
+
+    #[tokio::test]
+    async fn test_s3fifo_cache() {
+        case(s3fifo()).await
+    }
 }
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index d18bd18b..d1ba0c9d 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -104,6 +104,7 @@ pub trait Eviction: Send + Sync + 'static {
 pub mod fifo;
 pub mod lfu;
 pub mod lru;
+pub mod s3fifo;
 
 #[cfg(test)]
 pub mod test_utils;
diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs
new file mode 100644
index 00000000..c9f54e8a
--- /dev/null
+++ b/foyer-memory/src/eviction/s3fifo.rs
@@ -0,0 +1,420 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{fmt::Debug, ptr::NonNull};
+
+use foyer_intrusive::{
+    collections::dlist::{Dlist, DlistLink},
+    intrusive_adapter,
+};
+
+use crate::{
+    eviction::Eviction,
+    handle::{BaseHandle, Handle},
+    CacheContext, Key, Value,
+};
+
+#[derive(Debug, Clone)]
+pub struct S3FifoContext;
+
+impl From<CacheContext> for S3FifoContext {
+    fn from(_: CacheContext) -> Self {
+        Self
+    }
+}
+
+impl From<S3FifoContext> for CacheContext {
+    fn from(_: S3FifoContext) -> Self {
+        CacheContext::Default
+    }
+}
+
+enum Queue {
+    None,
+    Main,
+    Small,
+}
+
+pub struct S3FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    link: DlistLink,
+    base: BaseHandle<K, V, S3FifoContext>,
+    freq: u8,
+    queue: Queue,
+}
+
+impl<K, V> Debug for S3FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("S3FifoHandle").finish()
+    }
+}
+
+intrusive_adapter! { S3FifoHandleDlistAdapter<K, V> = NonNull<S3FifoHandle<K, V>>: S3FifoHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+
+impl<K, V> S3FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    #[inline(always)]
+    pub fn inc(&mut self) {
+        self.freq = std::cmp::min(self.freq + 1, 3);
+    }
+
+    #[inline(always)]
+    pub fn dec(&mut self) {
+        self.freq = self.freq.saturating_sub(1);
+    }
+
+    #[inline(always)]
+    pub fn reset(&mut self) {
+        self.freq = 0;
+    }
+}
+
+impl<K, V> Handle for S3FifoHandle<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Key = K;
+    type Value = V;
+    type Context = S3FifoContext;
+
+    fn new() -> Self {
+        Self {
+            link: DlistLink::default(),
+            freq: 0,
+            base: BaseHandle::new(),
+            queue: Queue::None,
+        }
+    }
+
+    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
+        self.base.init(hash, key, value, charge, context);
+    }
+
+    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &self.base
+    }
+
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+        &mut self.base
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct S3FifoConfig {
+    pub small_queue_capacity_ratio: f64,
+}
+
+pub struct S3Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    small_queue: Dlist<S3FifoHandleDlistAdapter<K, V>>,
+    main_queue: Dlist<S3FifoHandleDlistAdapter<K, V>>,
+
+    small_capacity: usize,
+
+    small_charges: usize,
+    main_charges: usize,
+}
+
+impl<K, V> S3Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    unsafe fn evict(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
+        if self.small_charges > self.small_capacity
+            && let Some(ptr) = self.evict_small()
+        {
+            Some(ptr)
+        } else {
+            self.evict_main()
+        }
+    }
+
+    unsafe fn evict_small(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
+        while let Some(mut ptr) = self.small_queue.pop_front() {
+            let handle = ptr.as_mut();
+            if handle.freq > 1 {
+                self.main_queue.push_back(ptr);
+                handle.queue = Queue::Main;
+                self.small_charges -= handle.base().charge();
+                self.main_charges += handle.base().charge();
+            } else {
+                handle.queue = Queue::None;
+                handle.reset();
+                self.small_charges -= handle.base().charge();
+                return Some(ptr);
+            }
+        }
+        None
+    }
+
+    unsafe fn evict_main(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
+        while let Some(mut ptr) = self.main_queue.pop_front() {
+            let handle = ptr.as_mut();
+            if handle.freq > 0 {
+                self.main_queue.push_back(ptr);
+                handle.dec();
+            } else {
+                handle.queue = Queue::None;
+                self.main_charges -= handle.base.charge();
+                return Some(ptr);
+            }
+        }
+        None
+    }
+}
+
+impl<K, V> Eviction for S3Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+    type Handle = S3FifoHandle<K, V>;
+    type Config = S3FifoConfig;
+
+    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
+    where
+        Self: Sized,
+    {
+        let small_capacity = (capacity as f64 * config.small_queue_capacity_ratio) as usize;
+        Self {
+            small_queue: Dlist::new(),
+            main_queue: Dlist::new(),
+            small_capacity,
+            small_charges: 0,
+            main_charges: 0,
+        }
+    }
+
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+
+        self.small_queue.push_back(ptr);
+        handle.queue = Queue::Small;
+        self.small_charges += handle.base().charge();
+
+        handle.base_mut().set_in_eviction(true);
+    }
+
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+        if let Some(mut ptr) = self.evict() {
+            let handle = ptr.as_mut();
+            // `handle.queue` has already been set with `evict()`
+            handle.base_mut().set_in_eviction(false);
+            Some(ptr)
+        } else {
+            debug_assert!(self.is_empty());
+            None
+        }
+    }
+
+    unsafe fn reinsert(&mut self, _: NonNull<Self::Handle>) {}
+
+    unsafe fn access(&mut self, ptr: NonNull<Self::Handle>) {
+        let mut ptr = ptr;
+        ptr.as_mut().inc();
+    }
+
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+        let handle = ptr.as_mut();
+
+        match handle.queue {
+            Queue::None => unreachable!(),
+            Queue::Main => {
+                let p = self
+                    .main_queue
+                    .iter_mut_from_raw(ptr.as_mut().link.raw())
+                    .remove()
+                    .unwrap_unchecked();
+                debug_assert_eq!(p, ptr);
+
+                handle.queue = Queue::None;
+                handle.base_mut().set_in_eviction(false);
+
+                self.main_charges -= handle.base().charge();
+            }
+            Queue::Small => {
+                let p = self
+                    .small_queue
+                    .iter_mut_from_raw(ptr.as_mut().link.raw())
+                    .remove()
+                    .unwrap_unchecked();
+                debug_assert_eq!(p, ptr);
+
+                handle.queue = Queue::None;
+                handle.base_mut().set_in_eviction(false);
+
+                self.small_charges -= handle.base().charge();
+            }
+        }
+    }
+
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+        let mut res = Vec::with_capacity(self.len());
+        while let Some(mut ptr) = self.small_queue.pop_front() {
+            let handle = ptr.as_mut();
+            handle.base_mut().set_in_eviction(false);
+            handle.queue = Queue::None;
+            res.push(ptr);
+        }
+        while let Some(mut ptr) = self.main_queue.pop_front() {
+            let handle = ptr.as_mut();
+            handle.base_mut().set_in_eviction(false);
+            handle.queue = Queue::None;
+            res.push(ptr);
+        }
+        res
+    }
+
+    unsafe fn len(&self) -> usize {
+        self.small_queue.len() + self.main_queue.len()
+    }
+
+    unsafe fn is_empty(&self) -> bool {
+        self.small_queue.is_empty() && self.main_queue.is_empty()
+    }
+}
+
+unsafe impl<K, V> Send for S3Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+unsafe impl<K, V> Sync for S3Fifo<K, V>
+where
+    K: Key,
+    V: Value,
+{
+}
+
+#[cfg(test)]
+mod tests {
+    use std::ops::Range;
+
+    use itertools::Itertools;
+
+    use super::*;
+    use crate::eviction::test_utils::TestEviction;
+
+    impl<K, V> TestEviction for S3Fifo<K, V>
+    where
+        K: Key + Clone,
+        V: Value + Clone,
+    {
+        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+            self.small_queue
+                .iter()
+                .chain(self.main_queue.iter())
+                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .collect_vec()
+        }
+    }
+
+    type TestS3Fifo = S3Fifo<u64, u64>;
+    type TestS3FifoHandle = S3FifoHandle<u64, u64>;
+
+    fn assert_test_s3fifo(s3fifo: &TestS3Fifo, small: Vec<u64>, main: Vec<u64>) {
+        let mut s = s3fifo
+            .dump()
+            .into_iter()
+            .map(|(k, v)| {
+                assert_eq!(k, v);
+                k
+            })
+            .collect_vec();
+        assert_eq!(s.len(), s3fifo.small_queue.len() + s3fifo.main_queue.len());
+        let m = s.split_off(s3fifo.small_queue.len());
+        assert_eq!((&s, &m), (&small, &main));
+        assert_eq!(s3fifo.small_charges, s.len());
+    }
+
+    fn assert_count(ptrs: &[NonNull<TestS3FifoHandle>], range: Range<usize>, count: u8) {
+        unsafe {
+            ptrs[range].iter().for_each(|ptr| assert_eq!(ptr.as_ref().freq, count));
+        }
+    }
+
+    #[test]
+    fn test_lfu() {
+        unsafe {
+            let ptrs = (0..100)
+                .map(|i| {
+                    let mut handle = Box::new(TestS3FifoHandle::new());
+                    handle.init(i, i, i, 1, S3FifoContext);
+                    NonNull::new_unchecked(Box::into_raw(handle))
+                })
+                .collect_vec();
+
+            // window: 2, probation: 2, protected: 6
+            let config = S3FifoConfig {
+                small_queue_capacity_ratio: 0.25,
+            };
+            let mut s3fifo = TestS3Fifo::new(8, &config);
+
+            assert_eq!(s3fifo.small_capacity, 2);
+
+            s3fifo.push(ptrs[0]);
+            s3fifo.push(ptrs[1]);
+            assert_test_s3fifo(&s3fifo, vec![0, 1], vec![]);
+
+            s3fifo.push(ptrs[2]);
+            s3fifo.push(ptrs[3]);
+            assert_test_s3fifo(&s3fifo, vec![0, 1, 2, 3], vec![]);
+
+            assert_count(&ptrs, 0..4, 0);
+
+            (0..4).for_each(|i| s3fifo.access(ptrs[i]));
+            s3fifo.access(ptrs[1]);
+            s3fifo.access(ptrs[2]);
+            assert_count(&ptrs, 0..1, 1);
+            assert_count(&ptrs, 1..3, 2);
+            assert_count(&ptrs, 3..4, 1);
+
+            let p0 = s3fifo.pop().unwrap();
+            let p3 = s3fifo.pop().unwrap();
+            assert_eq!(p0, ptrs[0]);
+            assert_eq!(p3, ptrs[3]);
+            assert_test_s3fifo(&s3fifo, vec![], vec![1, 2]);
+            assert_count(&ptrs, 0..1, 0);
+            assert_count(&ptrs, 1..3, 2);
+            assert_count(&ptrs, 3..4, 0);
+
+            let p1 = s3fifo.pop().unwrap();
+            assert_eq!(p1, ptrs[1]);
+            assert_test_s3fifo(&s3fifo, vec![], vec![2]);
+            assert_count(&ptrs, 0..4, 0);
+
+            assert_eq!(s3fifo.clear(), [2].into_iter().map(|i| ptrs[i]).collect_vec());
+
+            for ptr in ptrs {
+                let _ = Box::from_raw(ptr.as_ptr());
+            }
+        }
+    }
+}
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index a789b8d1..5ae7c4d0 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -13,9 +13,9 @@
 //  limitations under the License.
 
 pub use crate::{
-    cache::{Cache, CacheEntry, Entry, EntryState, FifoCacheConfig, LfuCacheConfig, LruCacheConfig},
+    cache::{Cache, CacheEntry, Entry, EntryState, FifoCacheConfig, LfuCacheConfig, LruCacheConfig, S3FifoCacheConfig},
     context::CacheContext,
-    eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig},
+    eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig, s3fifo::S3FifoConfig},
     listener::{CacheEventListener, DefaultCacheEventListener},
     metrics::Metrics,
 };
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 43e9182e..153f7282 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -28,6 +28,7 @@ futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
 hashbrown = { version = "0.14", features = ["raw"] }
+itertools = { version = "0.12" }
 libc = { version = "0.2", features = ["extra_traits"] }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
@@ -39,6 +40,7 @@ tracing-core = { version = "0.1" }
 [build-dependencies]
 cc = { version = "1", default-features = false, features = ["parallel"] }
 either = { version = "1", default-features = false, features = ["use_std"] }
+itertools = { version = "0.12" }
 proc-macro2 = { version = "1" }
 quote = { version = "1" }
 syn = { version = "2", features = ["extra-traits", "full", "visit-mut"] }

From cd612f67119eedb2fc243356eaa06950f39f4a10 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 8 Apr 2024 15:53:02 +0800
Subject: [PATCH 239/261] chore: bump foyer-intrusive to 0.3.1, foyer-memory to
 0.1.3 (#307)

* chore: bump foyer-memory to 0.1.3

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: bump foyer-intrusive to 0.3.1

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md               | 28 ++++++++++++++++++++++++++++
 foyer-intrusive/Cargo.toml |  2 +-
 foyer-memory/Cargo.toml    |  2 +-
 3 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 42893f24..68e3f3ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,31 @@
+## 2024-04-08
+
+| crate | version |
+| - | - |
+| foyer-intrusive | 0.3.1 |
+| foyer-memory | 0.1.3 |
+
+<details>
+
+### Changes
+
+- feat: Introduce s3fifo to `foyer-memory`.
+- fix: Fix doctest for `foyer-intrusive`.
+
+## 2024-03-21
+
+| crate | version |
+| - | - |
+| foyer-memory | 0.1.2 |
+
+<details>
+
+### Changes
+
+- fix: `foyer-memory` export `DefaultCacheEventListener`.
+
+</details>
+
 ## 2024-03-14
 
 | crate | version |
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 2d2618b4..f1356e72 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-intrusive"
-version = "0.3.0"
+version = "0.3.1"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "intrusive data structures for foyer - the hybrid cache for Rust"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index ffe808fd..9e51c7e1 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.1.2"
+version = "0.1.3"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"

From 5899ee4941f01714748fe4d7bd0be7bce3e66e17 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 9 Apr 2024 11:31:53 +0800
Subject: [PATCH 240/261] chore: bump foyer-storage to 0.5.1 (#309)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md             | 12 ++++++++++++
 foyer-storage/Cargo.toml |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68e3f3ae..fd7631c7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,15 @@
+## 2024-04-09
+
+| crate | version |
+| - | - |
+| foyer-storage | 0.5.1 |
+
+<details>
+
+### Changes
+
+- fix: Enable `offset_of` feature for `foyer-storage`.
+
 ## 2024-04-08
 
 | crate | version |
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 8e5b485a..b6069b8e 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.5.0"
+version = "0.5.1"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"

From b50813b94090e54c3d1df508978757356652f084 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 9 Apr 2024 22:31:37 +0800
Subject: [PATCH 241/261] fix: fix state() for s3fifo (#312)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs | 26 ++++++++++++++++----------
 foyer-memory/src/lib.rs   |  1 -
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index d1a59d69..269671e9 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -451,16 +451,22 @@ where
 {
     pub fn state(&self) -> EntryState {
         match self {
-            Entry::Fifo(FifoEntry::Hit(_)) | Entry::Lru(LruEntry::Hit(_)) | Entry::Lfu(LfuEntry::Hit(_)) => {
-                EntryState::Hit
-            }
-            Entry::Fifo(FifoEntry::Wait(_)) | Entry::Lru(LruEntry::Wait(_)) | Entry::Lfu(LfuEntry::Wait(_)) => {
-                EntryState::Wait
-            }
-            Entry::Fifo(FifoEntry::Miss(_)) | Entry::Lru(LruEntry::Miss(_)) | Entry::Lfu(LfuEntry::Miss(_)) => {
-                EntryState::Miss
-            }
-            _ => unreachable!(),
+            Entry::Fifo(FifoEntry::Hit(_))
+            | Entry::Lru(LruEntry::Hit(_))
+            | Entry::Lfu(LfuEntry::Hit(_))
+            | Entry::S3Fifo(S3FifoEntry::Hit(_)) => EntryState::Hit,
+            Entry::Fifo(FifoEntry::Wait(_))
+            | Entry::Lru(LruEntry::Wait(_))
+            | Entry::Lfu(LfuEntry::Wait(_))
+            | Entry::S3Fifo(S3FifoEntry::Wait(_)) => EntryState::Wait,
+            Entry::Fifo(FifoEntry::Miss(_))
+            | Entry::Lru(LruEntry::Miss(_))
+            | Entry::Lfu(LfuEntry::Miss(_))
+            | Entry::S3Fifo(S3FifoEntry::Miss(_)) => EntryState::Miss,
+            Entry::Fifo(FifoEntry::Invalid)
+            | Entry::Lru(LruEntry::Invalid)
+            | Entry::Lfu(LfuEntry::Invalid)
+            | Entry::S3Fifo(S3FifoEntry::Invalid) => unreachable!(),
         }
     }
 }
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index cc504e32..aa073eb3 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -65,7 +65,6 @@
 //! The handle that does not appear in either the indexer or the eviction container, and has no external owner, will be
 //! destroyed.
 
-#![feature(trait_alias)]
 #![feature(offset_of)]
 
 pub trait Key: Send + Sync + 'static + std::hash::Hash + Eq + Ord {}

From 5a410c8049bc81f92452125500a55cb10ababf0a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 9 Apr 2024 23:19:03 +0800
Subject: [PATCH 242/261] chore: bump foyer-memory to 0.1.4 (#313)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md            | 2 ++
 foyer-memory/Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd7631c7..2abd56b2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,11 +3,13 @@
 | crate | version |
 | - | - |
 | foyer-storage | 0.5.1 |
+| foyer-memory | 0.1.4 |
 
 <details>
 
 ### Changes
 
+- fix: Fix panics on `state()` for s3fifo entry.
 - fix: Enable `offset_of` feature for `foyer-storage`.
 
 ## 2024-04-08
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 9e51c7e1..2d151b69 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.1.3"
+version = "0.1.4"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"

From 823a8790c309ffac4150f745df60ea84e6d04ef0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 10 Apr 2024 14:49:00 +0800
Subject: [PATCH 243/261] refactor: use generic type for cursor instead of
 associated type (#314)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/code.rs   | 58 +++++++++++++++-----------------------
 foyer-common/src/lib.rs    |  1 -
 foyer-intrusive/src/lib.rs |  1 -
 foyer-storage/src/lib.rs   |  1 -
 4 files changed, 23 insertions(+), 38 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 6e127119..3e5c3ba3 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::marker::PhantomData;
+use std::{marker::PhantomData, sync::Arc};
 
 use bytes::{Buf, BufMut};
 use paste::paste;
@@ -88,20 +88,18 @@ trait BufMutExt: BufMut {
 
 impl<T: BufMut> BufMutExt for T {}
 
-pub trait Cursor: Send + Sync + 'static + std::io::Read + std::fmt::Debug {
-    type T: Send + Sync + 'static;
-
-    fn into_inner(self) -> Self::T;
+pub trait Cursor<T>: Send + Sync + 'static + std::io::Read + std::fmt::Debug {
+    fn into_inner(self) -> T;
 }
 
 /// [`Key`] is required to implement [`Clone`].
 ///
-/// If cloning a [`Key`] is expensive, wrap it with [`std::sync::Arc`].
+/// If cloning a [`Key`] is expensive, wrap it with [`Arc`].
 #[expect(unused_variables)]
 pub trait Key:
     Sized + Send + Sync + 'static + std::hash::Hash + Eq + PartialEq + Ord + PartialOrd + std::fmt::Debug + Clone
 {
-    type Cursor: Cursor<T = Self> = UnimplementedCursor<Self>;
+    type Cursor: Cursor<Self> = UnimplementedCursor<Self>;
 
     /// memory weight
     fn weight(&self) -> usize {
@@ -123,10 +121,10 @@ pub trait Key:
 
 /// [`Value`] is required to implement [`Clone`].
 ///
-/// If cloning a [`Value`] is expensive, wrap it with [`std::sync::Arc`].
+/// If cloning a [`Value`] is expensive, wrap it with [`Arc`].
 #[expect(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug + Clone {
-    type Cursor: Cursor<T = Self> = UnimplementedCursor<Self>;
+    type Cursor: Cursor<Self> = UnimplementedCursor<Self>;
 
     /// memory weight
     fn weight(&self) -> usize {
@@ -192,10 +190,8 @@ macro_rules! def_cursor {
                     }
                 }
 
-                impl Cursor for [<PrimitiveCursor $id>] {
-                    type T = $type;
-
-                    fn into_inner(self) -> Self::T {
+                impl Cursor<$type> for [<PrimitiveCursor $id>] {
+                    fn into_inner(self) -> $type {
                         self.inner
                     }
                 }
@@ -296,15 +292,13 @@ impl Value for Vec<u8> {
     }
 }
 
-impl Cursor for std::io::Cursor<Vec<u8>> {
-    type T = Vec<u8>;
-
-    fn into_inner(self) -> Self::T {
+impl Cursor<Vec<u8>> for std::io::Cursor<Vec<u8>> {
+    fn into_inner(self) -> Vec<u8> {
         self.into_inner()
     }
 }
 
-impl Key for std::sync::Arc<Vec<u8>> {
+impl Key for Arc<Vec<u8>> {
     type Cursor = ArcVecU8Cursor;
 
     fn weight(&self) -> usize {
@@ -316,7 +310,7 @@ impl Key for std::sync::Arc<Vec<u8>> {
     }
 
     fn read(buf: &[u8]) -> CodingResult<Self> {
-        Ok(std::sync::Arc::new(buf.to_vec()))
+        Ok(Arc::new(buf.to_vec()))
     }
 
     fn into_cursor(self) -> Self::Cursor {
@@ -324,7 +318,7 @@ impl Key for std::sync::Arc<Vec<u8>> {
     }
 }
 
-impl Value for std::sync::Arc<Vec<u8>> {
+impl Value for Arc<Vec<u8>> {
     type Cursor = ArcVecU8Cursor;
 
     fn weight(&self) -> usize {
@@ -336,7 +330,7 @@ impl Value for std::sync::Arc<Vec<u8>> {
     }
 
     fn read(buf: &[u8]) -> CodingResult<Self> {
-        Ok(std::sync::Arc::new(buf.to_vec()))
+        Ok(Arc::new(buf.to_vec()))
     }
 
     fn into_cursor(self) -> Self::Cursor {
@@ -346,12 +340,12 @@ impl Value for std::sync::Arc<Vec<u8>> {
 
 #[derive(Debug)]
 pub struct ArcVecU8Cursor {
-    inner: std::sync::Arc<Vec<u8>>,
+    inner: Arc<Vec<u8>>,
     pos: usize,
 }
 
 impl ArcVecU8Cursor {
-    pub fn new(inner: std::sync::Arc<Vec<u8>>) -> Self {
+    pub fn new(inner: Arc<Vec<u8>>) -> Self {
         Self { inner, pos: 0 }
     }
 }
@@ -366,10 +360,8 @@ impl std::io::Read for ArcVecU8Cursor {
     }
 }
 
-impl Cursor for ArcVecU8Cursor {
-    type T = std::sync::Arc<Vec<u8>>;
-
-    fn into_inner(self) -> Self::T {
+impl Cursor<Arc<Vec<u8>>> for ArcVecU8Cursor {
+    fn into_inner(self) -> Arc<Vec<u8>> {
         self.inner
     }
 }
@@ -383,10 +375,8 @@ impl std::io::Read for PrimitiveCursorVoid {
     }
 }
 
-impl Cursor for PrimitiveCursorVoid {
-    type T = ();
-
-    fn into_inner(self) -> Self::T {}
+impl Cursor<()> for PrimitiveCursorVoid {
+    fn into_inner(self) {}
 }
 
 impl Key for () {
@@ -438,10 +428,8 @@ impl<T: Send + Sync + 'static + std::fmt::Debug> std::io::Read for Unimplemented
     }
 }
 
-impl<T: Send + Sync + 'static + std::fmt::Debug> Cursor for UnimplementedCursor<T> {
-    type T = T;
-
-    fn into_inner(self) -> Self::T {
+impl<T: Send + Sync + 'static + std::fmt::Debug> Cursor<T> for UnimplementedCursor<T> {
+    fn into_inner(self) -> T {
         unimplemented!()
     }
 }
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 46e94ee4..1681a227 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -17,7 +17,6 @@
 #![feature(bound_map)]
 #![feature(associated_type_defaults)]
 #![feature(cfg_match)]
-#![feature(let_chains)]
 #![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 
 pub mod async_queue;
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 4241d821..074ed6ba 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -12,7 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(associated_type_bounds)]
 #![feature(ptr_metadata)]
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index d9e98496..2799f01b 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -20,7 +20,6 @@
 #![feature(error_generic_member_access)]
 #![feature(lazy_cell)]
 #![feature(lint_reasons)]
-#![feature(associated_type_defaults)]
 #![feature(box_into_inner)]
 #![feature(try_trait_v2)]
 #![feature(offset_of)]

From 3aa689d69845ec52a4459463d9fc6daf3d910bf4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 10 Apr 2024 15:09:49 +0800
Subject: [PATCH 244/261] refactor: remove unstable feature let_chains and
 lint_reasons (#315)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/code.rs                |  6 ++++--
 foyer-common/src/lib.rs                 |  1 -
 foyer-experimental/src/lib.rs           |  1 -
 foyer-intrusive/src/lib.rs              |  4 ++--
 foyer-memory/src/eviction/s3fifo.rs     | 12 ++++++------
 foyer-memory/src/eviction/test_utils.rs |  3 ++-
 foyer-memory/src/generic.rs             | 18 ++++++++++++------
 foyer-memory/src/lib.rs                 |  3 ---
 foyer-storage-bench/src/main.rs         | 23 +++++++++++------------
 foyer-storage/src/admission/mod.rs      |  3 +--
 foyer-storage/src/buffer.rs             |  3 ++-
 foyer-storage/src/catalog.rs            | 22 ++++++++++++----------
 foyer-storage/src/generic.rs            |  6 ++++--
 foyer-storage/src/lib.rs                |  2 --
 foyer-storage/src/region.rs             |  9 ++++++---
 foyer-storage/src/reinsertion/mod.rs    |  3 +--
 foyer-storage/src/test_utils.rs         |  9 ++++++++-
 foyer-storage/tests/storage_test.rs     |  4 ++--
 18 files changed, 73 insertions(+), 59 deletions(-)

diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index 3e5c3ba3..dda8e00a 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -95,7 +95,8 @@ pub trait Cursor<T>: Send + Sync + 'static + std::io::Read + std::fmt::Debug {
 /// [`Key`] is required to implement [`Clone`].
 ///
 /// If cloning a [`Key`] is expensive, wrap it with [`Arc`].
-#[expect(unused_variables)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[allow(unused_variables)]
 pub trait Key:
     Sized + Send + Sync + 'static + std::hash::Hash + Eq + PartialEq + Ord + PartialOrd + std::fmt::Debug + Clone
 {
@@ -122,7 +123,8 @@ pub trait Key:
 /// [`Value`] is required to implement [`Clone`].
 ///
 /// If cloning a [`Value`] is expensive, wrap it with [`Arc`].
-#[expect(unused_variables)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[allow(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug + Clone {
     type Cursor: Cursor<Self> = UnimplementedCursor<Self>;
 
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 1681a227..a9541ec9 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -13,7 +13,6 @@
 //  limitations under the License.
 
 #![feature(trait_alias)]
-#![feature(lint_reasons)]
 #![feature(bound_map)]
 #![feature(associated_type_defaults)]
 #![feature(cfg_match)]
diff --git a/foyer-experimental/src/lib.rs b/foyer-experimental/src/lib.rs
index 1d8ca5e2..ec80631e 100644
--- a/foyer-experimental/src/lib.rs
+++ b/foyer-experimental/src/lib.rs
@@ -13,7 +13,6 @@
 //  limitations under the License.
 
 #![feature(cfg_match)]
-#![feature(lint_reasons)]
 #![feature(error_generic_member_access)]
 #![feature(lazy_cell)]
 
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 074ed6ba..6d6fbc28 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -14,9 +14,9 @@
 
 #![feature(ptr_metadata)]
 #![feature(trait_alias)]
-#![feature(lint_reasons)]
 #![feature(offset_of)]
-#![expect(clippy::new_without_default)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#![allow(clippy::new_without_default)]
 
 pub use memoffset::offset_of;
 
diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs
index c9f54e8a..e8911494 100644
--- a/foyer-memory/src/eviction/s3fifo.rs
+++ b/foyer-memory/src/eviction/s3fifo.rs
@@ -146,13 +146,13 @@ where
     V: Value,
 {
     unsafe fn evict(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
-        if self.small_charges > self.small_capacity
-            && let Some(ptr) = self.evict_small()
-        {
-            Some(ptr)
-        } else {
-            self.evict_main()
+        // TODO(MrCroxx): Use `let_chains` here after it is stable.
+        if self.small_charges > self.small_capacity {
+            if let Some(ptr) = self.evict_small() {
+                return Some(ptr);
+            }
         }
+        self.evict_main()
     }
 
     unsafe fn evict_small(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
diff --git a/foyer-memory/src/eviction/test_utils.rs b/foyer-memory/src/eviction/test_utils.rs
index 39b57347..adf478de 100644
--- a/foyer-memory/src/eviction/test_utils.rs
+++ b/foyer-memory/src/eviction/test_utils.rs
@@ -15,7 +15,8 @@
 use super::Eviction;
 use crate::handle::Handle;
 
-#[expect(clippy::type_complexity)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[allow(clippy::type_complexity)]
 pub trait TestEviction: Eviction {
     fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)>;
 }
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index 6352c368..5ac43267 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -43,7 +43,8 @@ struct CacheSharedState<T, L> {
     listener: L,
 }
 
-#[expect(clippy::type_complexity)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[allow(clippy::type_complexity)]
 struct CacheShard<K, V, H, E, I, L, S>
 where
     K: Key,
@@ -198,9 +199,12 @@ where
     }
 
     unsafe fn evict(&mut self, charge: usize, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
-        while self.usage.load(Ordering::Relaxed) + charge > self.capacity
-            && let Some(evicted) = self.eviction.pop()
-        {
+        // TODO(MrCroxx): Use `let_chains` here after it is stable.
+        while self.usage.load(Ordering::Relaxed) + charge > self.capacity {
+            let evicted = match self.eviction.pop() {
+                Some(evicted) => evicted,
+                None => break,
+            };
             self.state.metrics.evict.fetch_add(1, Ordering::Relaxed);
             let base = evicted.as_ref().base();
             debug_assert!(base.is_in_indexer());
@@ -304,7 +308,8 @@ where
     pub event_listener: L,
 }
 
-#[expect(clippy::type_complexity)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[allow(clippy::type_complexity)]
 pub enum GenericEntry<K, V, H, E, I, L, S, ER>
 where
     K: Key,
@@ -364,7 +369,8 @@ where
     }
 }
 
-#[expect(clippy::type_complexity)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[allow(clippy::type_complexity)]
 pub struct GenericCache<K, V, H, E, I, L, S = RandomState>
 where
     K: Key,
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index aa073eb3..3b92ad5a 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -12,9 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(let_chains)]
-#![feature(lint_reasons)]
-
 //! This crate provides a concurrent in-memory cache component that supports replaceable eviction algorithm.
 //!
 //! # Motivation
diff --git a/foyer-storage-bench/src/main.rs b/foyer-storage-bench/src/main.rs
index c2efeb34..acd59aab 100644
--- a/foyer-storage-bench/src/main.rs
+++ b/foyer-storage-bench/src/main.rs
@@ -12,9 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(let_chains)]
-#![feature(lint_reasons)]
-
 mod analyze;
 mod export;
 mod rate;
@@ -754,13 +751,14 @@ async fn write(
         }
 
         let idx = id + step * c;
-        // TODO(MrCroxx): Use random content?
         let entry_size = OsRng.gen_range(context.entry_size_range.clone());
         let data = Arc::new(text(idx as usize, entry_size));
-        if let Some(limiter) = &mut limiter
-            && let Some(wait) = limiter.consume(entry_size as f64)
-        {
-            tokio::time::sleep(wait).await;
+
+        // TODO(MrCroxx): Use `let_chains` here after it is stable.
+        if let Some(limiter) = &mut limiter {
+            if let Some(wait) = limiter.consume(entry_size as f64) {
+                tokio::time::sleep(wait).await;
+            }
         }
 
         let time = Instant::now();
@@ -849,10 +847,11 @@ async fn read(
             }
             context.metrics.get_bytes.fetch_add(entry_size, Ordering::Relaxed);
 
-            if let Some(limiter) = &mut limiter
-                && let Some(wait) = limiter.consume(entry_size as f64)
-            {
-                tokio::time::sleep(wait).await;
+            // TODO(MrCroxx): Use `let_chains` here after it is stable.
+            if let Some(limiter) = &mut limiter {
+                if let Some(wait) = limiter.consume(entry_size as f64) {
+                    tokio::time::sleep(wait).await;
+                }
             }
         } else {
             if let Err(e) = context.metrics.get_miss_lats.write().record(lat) {
diff --git a/foyer-storage/src/admission/mod.rs b/foyer-storage/src/admission/mod.rs
index dcff9e98..406ed732 100644
--- a/foyer-storage/src/admission/mod.rs
+++ b/foyer-storage/src/admission/mod.rs
@@ -41,12 +41,11 @@ where
     }
 }
 
-#[expect(unused_variables)]
 pub trait AdmissionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, context: AdmissionContext<Self::Key, Self::Value>) {}
+    fn init(&self, context: AdmissionContext<Self::Key, Self::Value>);
 
     fn judge(&self, key: &Self::Key, weight: usize) -> bool;
 
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index dfc0c179..8090ef7e 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -179,7 +179,8 @@ where
     /// # Format
     ///
     /// | header | value (compressed) | key | <padding> |
-    #[expect(clippy::uninit_vec)]
+    // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+    #[allow(clippy::uninit_vec)]
     pub async fn write(
         &mut self,
         Entry {
diff --git a/foyer-storage/src/catalog.rs b/foyer-storage/src/catalog.rs
index 3177995b..bbc9c749 100644
--- a/foyer-storage/src/catalog.rs
+++ b/foyer-storage/src/catalog.rs
@@ -130,12 +130,13 @@ where
             item.inserted = Some(Instant::now());
             guard.insert(key.clone(), item)
         };
-        if let Some(old) = old
-            && let Index::Inflight { .. } = old.index()
-        {
-            self.metrics
-                .inner_op_duration_entry_flush
-                .observe(old.inserted.unwrap().elapsed().as_secs_f64());
+        // TODO(MrCroxx): Use `let_chains` here after it is stable.
+        if let Some(old) = old {
+            if let Index::Inflight { .. } = old.index() {
+                self.metrics
+                    .inner_op_duration_entry_flush
+                    .observe(old.inserted.unwrap().elapsed().as_secs_f64());
+            }
         }
     }
 
@@ -147,10 +148,11 @@ where
     pub fn remove(&self, key: &K) -> Option<Item<K, V>> {
         let shard = self.shard(key);
         let info: Option<Item<K, V>> = self.items[shard].write().remove(key);
-        if let Some(info) = &info
-            && let Index::Region { view } = &info.index
-        {
-            self.regions[*view.id() as usize].lock().remove(key);
+        // TODO(MrCroxx): Use `let_chains` here after it is stable.
+        if let Some(info) = &info {
+            if let Index::Region { view } = &info.index {
+                self.regions[*view.id() as usize].lock().remove(key);
+            }
         }
         info
     }
diff --git a/foyer-storage/src/generic.rs b/foyer-storage/src/generic.rs
index 490c3915..51522814 100644
--- a/foyer-storage/src/generic.rs
+++ b/foyer-storage/src/generic.rs
@@ -236,7 +236,8 @@ where
 
         let (flushers_stop_tx, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY);
         let flusher_stop_rxs = (0..config.flushers).map(|_| flushers_stop_tx.subscribe()).collect_vec();
-        #[expect(clippy::type_complexity)]
+        // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+        #[allow(clippy::type_complexity)]
         let (flusher_entry_txs, flusher_entry_rxs): (
             Vec<mpsc::UnboundedSender<Entry<K, V>>>,
             Vec<mpsc::UnboundedReceiver<Entry<K, V>>>,
@@ -1081,7 +1082,8 @@ mod tests {
     type TestStoreConfig = GenericStoreConfig<u64, Vec<u8>, FsDevice, Fifo<RegionEpItemAdapter<FifoLink>>>;
 
     #[tokio::test]
-    #[expect(clippy::identity_op)]
+    // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+    #[allow(clippy::identity_op)]
     async fn test_recovery() {
         const KB: usize = 1024;
         const MB: usize = 1024 * 1024;
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 2799f01b..7e2cf9a1 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -16,10 +16,8 @@
 #![feature(strict_provenance)]
 #![feature(trait_alias)]
 #![feature(get_mut_unchecked)]
-#![feature(let_chains)]
 #![feature(error_generic_member_access)]
 #![feature(lazy_cell)]
-#![feature(lint_reasons)]
 #![feature(box_into_inner)]
 #![feature(try_trait_v2)]
 #![feature(offset_of)]
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 94a7dc37..4280d395 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -105,7 +105,8 @@ pub struct RegionInner<A>
 where
     A: BufferAllocator,
 {
-    #[expect(clippy::type_complexity)]
+    // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+    #[allow(clippy::type_complexity)]
     waits: BTreeMap<(usize, usize), Vec<oneshot::Sender<Result<Arc<Vec<u8, A>>>>>>,
 }
 
@@ -152,7 +153,8 @@ where
     }
 
     /// Load region data by view from device.
-    #[expect(clippy::type_complexity)]
+    // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+    #[allow(clippy::type_complexity)]
     #[tracing::instrument(skip(self, view))]
     pub async fn load(&self, view: RegionView) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
         let res = self
@@ -164,7 +166,8 @@ where
     }
 
     /// Load region data with given `range` from device.
-    #[expect(clippy::type_complexity)]
+    // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+    #[allow(clippy::type_complexity)]
     #[tracing::instrument(skip(self, range), fields(start, end))]
     pub async fn load_range(
         &self,
diff --git a/foyer-storage/src/reinsertion/mod.rs b/foyer-storage/src/reinsertion/mod.rs
index 493c941f..7681863e 100644
--- a/foyer-storage/src/reinsertion/mod.rs
+++ b/foyer-storage/src/reinsertion/mod.rs
@@ -41,12 +41,11 @@ where
     }
 }
 
-#[expect(unused_variables)]
 pub trait ReinsertionPolicy: Send + Sync + 'static + Debug {
     type Key: Key;
     type Value: Value;
 
-    fn init(&self, context: ReinsertionContext<Self::Key, Self::Value>) {}
+    fn init(&self, context: ReinsertionContext<Self::Key, Self::Value>);
 
     fn judge(&self, key: &Self::Key, weight: usize) -> bool;
 
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index 25fffc85..1c155ca0 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -17,7 +17,10 @@ use std::{collections::HashSet, marker::PhantomData};
 use foyer_common::code::{Key, Value};
 use parking_lot::Mutex;
 
-use crate::{admission::AdmissionPolicy, reinsertion::ReinsertionPolicy};
+use crate::{
+    admission::{AdmissionContext, AdmissionPolicy},
+    reinsertion::{ReinsertionContext, ReinsertionPolicy},
+};
 
 #[derive(Debug, Clone)]
 pub enum Record<K: Key> {
@@ -83,6 +86,8 @@ where
 
     type Value = V;
 
+    fn init(&self, _: AdmissionContext<Self::Key, Self::Value>) {}
+
     fn judge(&self, key: &K, _weight: usize) -> bool {
         self.records.lock().push(Record::Admit(key.clone()));
         true
@@ -102,6 +107,8 @@ where
 
     type Value = V;
 
+    fn init(&self, _: ReinsertionContext<Self::Key, Self::Value>) {}
+
     fn judge(&self, key: &K, _weight: usize) -> bool {
         self.records.lock().push(Record::Evict(key.clone()));
         false
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index 97bc7143..3910f881 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -12,8 +12,8 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(lint_reasons)]
-#![expect(clippy::identity_op)]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#![allow(clippy::identity_op)]
 
 use std::{path::PathBuf, sync::Arc, time::Duration};
 

From 269366957b97c2df0ae1d09b14c0d967a2731300 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 10 Apr 2024 16:10:14 +0800
Subject: [PATCH 245/261] refactor: remove usage of unstable features (#316)

* refactor: remove usage of unstable features

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: some more

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/Cargo.toml               |  1 +
 foyer-common/src/bits.rs              | 38 +++++++++--
 foyer-common/src/code.rs              | 94 ++++++++++++++++++++++-----
 foyer-common/src/lib.rs               |  3 -
 foyer-common/src/range.rs             | 19 ++++--
 foyer-experimental/Cargo.toml         |  3 +-
 foyer-experimental/src/error.rs       |  5 +-
 foyer-experimental/src/lib.rs         |  5 --
 foyer-experimental/src/metrics.rs     |  9 ++-
 foyer-experimental/src/wal.rs         |  8 +--
 foyer-intrusive/src/eviction/mod.rs   |  4 +-
 foyer-intrusive/src/lib.rs            |  2 -
 foyer-storage/Cargo.toml              |  1 +
 foyer-storage/src/buffer.rs           |  4 +-
 foyer-storage/src/device/allocator.rs |  6 +-
 foyer-storage/src/device/mod.rs       | 19 ++++--
 foyer-storage/src/flusher.rs          |  2 +-
 foyer-storage/src/lib.rs              |  7 --
 foyer-storage/src/metrics.rs          | 11 +++-
 foyer-storage/src/storage.rs          |  6 +-
 foyer/src/lib.rs                      |  3 -
 21 files changed, 173 insertions(+), 77 deletions(-)

diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 16a298b5..0af9293f 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -15,6 +15,7 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 anyhow = "1.0"
 bytes = "1"
+cfg-if = "1"
 foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
diff --git a/foyer-common/src/bits.rs b/foyer-common/src/bits.rs
index c7bc8aee..245e02be 100644
--- a/foyer-common/src/bits.rs
+++ b/foyer-common/src/bits.rs
@@ -31,7 +31,21 @@ use std::{
     ops::{Add, BitAnd, Not, Sub},
 };
 
-pub trait UnsignedTrait = Add<Output = Self>
+// TODO(MrCroxx): Use `trait_alias` after stable.
+// pub trait UnsignedTrait = Add<Output = Self>
+//     + Sub<Output = Self>
+//     + BitAnd<Output = Self>
+//     + Not<Output = Self>
+//     + Sized
+//     + From<u8>
+//     + Eq
+//     + Debug
+//     + Display
+//     + Clone
+//     + Copy;
+
+pub trait Unsigned:
+    Add<Output = Self>
     + Sub<Output = Self>
     + BitAnd<Output = Self>
     + Not<Output = Self>
@@ -41,11 +55,25 @@ pub trait UnsignedTrait = Add<Output = Self>
     + Debug
     + Display
     + Clone
-    + Copy;
-
-pub trait Unsigned: UnsignedTrait {}
+    + Copy
+{
+}
 
-impl<U: UnsignedTrait> Unsigned for U {}
+impl<
+        U: Add<Output = Self>
+            + Sub<Output = Self>
+            + BitAnd<Output = Self>
+            + Not<Output = Self>
+            + Sized
+            + From<u8>
+            + Eq
+            + Debug
+            + Display
+            + Clone
+            + Copy,
+    > Unsigned for U
+{
+}
 
 #[inline(always)]
 pub fn is_pow2<U: Unsigned>(v: U) -> bool {
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index dda8e00a..22d35d67 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -20,31 +20,58 @@ use paste::paste;
 pub type CodingError = anyhow::Error;
 pub type CodingResult<T> = Result<T, CodingError>;
 
-trait BufExt: Buf {
-    cfg_match! {
-        cfg(target_pointer_width = "16") => {
+pub trait BufExt: Buf {
+    // TODO(MrCroxx): Use `cfg_match` after stable.
+    // cfg_match! {
+    //     cfg(target_pointer_width = "16") => {
+    //         fn get_usize(&mut self) -> usize {
+    //             self.get_u16() as usize
+    //         }
+
+    //         fn get_isize(&mut self) -> isize {
+    //             self.get_i16() as isize
+    //         }
+    //     }
+    //     cfg(target_pointer_width = "32") => {
+    //         fn get_usize(&mut self) -> usize {
+    //             self.get_u32() as usize
+    //         }
+
+    //         fn get_isize(&mut self) -> isize {
+    //             self.get_i32() as isize
+    //         }
+    //     }
+    //     cfg(target_pointer_width = "64") => {
+    //         fn get_usize(&mut self) -> usize {
+    //             self.get_u64() as usize
+    //         }
+
+    //         fn get_isize(&mut self) -> isize {
+    //             self.get_i64() as isize
+    //         }
+    //     }
+    // }
+    cfg_if::cfg_if! {
+        if #[cfg(target_pointer_width = "16")] {
             fn get_usize(&mut self) -> usize {
                 self.get_u16() as usize
             }
-
             fn get_isize(&mut self) -> isize {
                 self.get_i16() as isize
             }
         }
-        cfg(target_pointer_width = "32") => {
+        else if #[cfg(target_pointer_width = "32")] {
             fn get_usize(&mut self) -> usize {
                 self.get_u32() as usize
             }
-
             fn get_isize(&mut self) -> isize {
                 self.get_i32() as isize
             }
         }
-        cfg(target_pointer_width = "64") => {
+        else if #[cfg(target_pointer_width = "64")] {
             fn get_usize(&mut self) -> usize {
                 self.get_u64() as usize
             }
-
             fn get_isize(&mut self) -> isize {
                 self.get_i64() as isize
             }
@@ -54,31 +81,58 @@ trait BufExt: Buf {
 
 impl<T: Buf> BufExt for T {}
 
-trait BufMutExt: BufMut {
-    cfg_match! {
-        cfg(target_pointer_width = "16") => {
+pub trait BufMutExt: BufMut {
+    // TODO(MrCroxx): Use `cfg_match` after stable.
+    // cfg_match! {
+    //     cfg(target_pointer_width = "16") => {
+    //         fn put_usize(&mut self, v: usize) {
+    //             self.put_u16(v as u16);
+    //         }
+
+    //         fn put_isize(&mut self, v: isize) {
+    //             self.put_i16(v as i16);
+    //         }
+    //     }
+    //     cfg(target_pointer_width = "32") => {
+    //         fn put_usize(&mut self, v: usize) {
+    //             self.put_u32(v as u32);
+    //         }
+
+    //         fn put_isize(&mut self, v: isize) {
+    //             self.put_i32(v as i32);
+    //         }
+    //     }
+    //     cfg(target_pointer_width = "64") => {
+    //         fn put_usize(&mut self, v: usize) {
+    //             self.put_u64(v as u64);
+    //         }
+
+    //         fn put_isize(&mut self, v: isize) {
+    //             self.put_i64(v as i64);
+    //         }
+    //     }
+    // }
+    cfg_if::cfg_if! {
+        if #[cfg(target_pointer_width = "16")] {
             fn put_usize(&mut self, v: usize) {
                 self.put_u16(v as u16);
             }
-
             fn put_isize(&mut self, v: isize) {
                 self.put_i16(v as i16);
             }
         }
-        cfg(target_pointer_width = "32") => {
+        else if #[cfg(target_pointer_width = "32")] {
             fn put_usize(&mut self, v: usize) {
                 self.put_u32(v as u32);
             }
-
             fn put_isize(&mut self, v: isize) {
                 self.put_i32(v as i32);
             }
         }
-        cfg(target_pointer_width = "64") => {
+        else if #[cfg(target_pointer_width = "64")] {
             fn put_usize(&mut self, v: usize) {
                 self.put_u64(v as u64);
             }
-
             fn put_isize(&mut self, v: isize) {
                 self.put_i64(v as i64);
             }
@@ -100,7 +154,9 @@ pub trait Cursor<T>: Send + Sync + 'static + std::io::Read + std::fmt::Debug {
 pub trait Key:
     Sized + Send + Sync + 'static + std::hash::Hash + Eq + PartialEq + Ord + PartialOrd + std::fmt::Debug + Clone
 {
-    type Cursor: Cursor<Self> = UnimplementedCursor<Self>;
+    // TODO(MrCroxx): Restore this after `associated_type_defaults` is stable.
+    // type Cursor: Cursor<Self> = UnimplementedCursor<Self>;
+    type Cursor: Cursor<Self>;
 
     /// memory weight
     fn weight(&self) -> usize {
@@ -126,7 +182,9 @@ pub trait Key:
 // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
 #[allow(unused_variables)]
 pub trait Value: Sized + Send + Sync + 'static + std::fmt::Debug + Clone {
-    type Cursor: Cursor<Self> = UnimplementedCursor<Self>;
+    // TODO(MrCroxx): Restore this after `associated_type_defaults` is stable.
+    // type Cursor: Cursor<Self> = UnimplementedCursor<Self>;
+    type Cursor: Cursor<Self>;
 
     /// memory weight
     fn weight(&self) -> usize {
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index a9541ec9..735c635e 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -12,10 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(trait_alias)]
 #![feature(bound_map)]
-#![feature(associated_type_defaults)]
-#![feature(cfg_match)]
 #![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 
 pub mod async_queue;
diff --git a/foyer-common/src/range.rs b/foyer-common/src/range.rs
index bb107413..0e887b3f 100644
--- a/foyer-common/src/range.rs
+++ b/foyer-common/src/range.rs
@@ -48,10 +48,14 @@ mod private {
 
 use private::ZeroOne;
 
-pub trait Idx =
-    PartialOrd<Self> + Add<Output = Self> + Sub<Output = Self> + Clone + Copy + Send + Sync + 'static + ZeroOne;
-
-pub trait RangeBoundsExt<T: Idx>: RangeBounds<T> {
+// TODO(MrCroxx): Use `trait_alias` after stable.
+// pub trait Idx =
+//     PartialOrd<Self> + Add<Output = Self> + Sub<Output = Self> + Clone + Copy + Send + Sync + 'static + ZeroOne;
+
+pub trait RangeBoundsExt<
+    T: PartialOrd<T> + Add<Output = T> + Sub<Output = T> + Clone + Copy + Send + Sync + 'static + ZeroOne,
+>: RangeBounds<T>
+{
     fn start(&self) -> Option<T> {
         match self.start_bound() {
             Bound::Included(v) => Some(*v),
@@ -107,4 +111,9 @@ pub trait RangeBoundsExt<T: Idx>: RangeBounds<T> {
     }
 }
 
-impl<T: Idx, RB: RangeBounds<T>> RangeBoundsExt<T> for RB {}
+impl<
+        T: PartialOrd<T> + Add<Output = T> + Sub<Output = T> + Clone + Copy + Send + Sync + 'static + ZeroOne,
+        RB: RangeBounds<T>,
+    > RangeBoundsExt<T> for RB
+{
+}
diff --git a/foyer-experimental/Cargo.toml b/foyer-experimental/Cargo.toml
index 538389c6..48bdc161 100644
--- a/foyer-experimental/Cargo.toml
+++ b/foyer-experimental/Cargo.toml
@@ -16,7 +16,9 @@ normal = ["foyer-workspace-hack"]
 anyhow = "1.0"
 bytes = "1"
 crossbeam = { version = "0.8", features = ["std", "crossbeam-channel"] }
+foyer-common = { version = "0.4", path = "../foyer-common" }
 foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+lazy_static = "1"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
 prometheus = "0.13"
@@ -27,7 +29,6 @@ tracing = "0.1"
 [dev-dependencies]
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
-foyer-common = { version = "0.4", path = "../foyer-common" }
 hdrhistogram = "7"
 itertools = "0.12"
 rand = "0.8.5"
diff --git a/foyer-experimental/src/error.rs b/foyer-experimental/src/error.rs
index cb50531c..383930e4 100644
--- a/foyer-experimental/src/error.rs
+++ b/foyer-experimental/src/error.rs
@@ -12,8 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::backtrace::Backtrace;
-
 #[derive(thiserror::Error, Debug)]
 #[error("{0}")]
 pub struct Error(Box<ErrorInner>);
@@ -23,7 +21,8 @@ pub struct Error(Box<ErrorInner>);
 struct ErrorInner {
     #[from]
     source: ErrorKind,
-    backtrace: Backtrace,
+    // TODO(MrCroxx): Restore this after `error_generic_member_access` is stable.
+    // backtrace: Backtrace,
 }
 
 #[derive(thiserror::Error, Debug)]
diff --git a/foyer-experimental/src/lib.rs b/foyer-experimental/src/lib.rs
index ec80631e..c8a1dd61 100644
--- a/foyer-experimental/src/lib.rs
+++ b/foyer-experimental/src/lib.rs
@@ -12,11 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(cfg_match)]
-#![feature(error_generic_member_access)]
-#![feature(lazy_cell)]
-
-pub mod buf;
 pub mod error;
 pub mod metrics;
 pub mod notify;
diff --git a/foyer-experimental/src/metrics.rs b/foyer-experimental/src/metrics.rs
index 5b07df73..8a15e4ec 100644
--- a/foyer-experimental/src/metrics.rs
+++ b/foyer-experimental/src/metrics.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::{LazyLock, OnceLock};
+use std::sync::OnceLock;
 
 use prometheus::{
     core::{AtomicU64, GenericGauge, GenericGaugeVec},
@@ -55,7 +55,12 @@ pub fn get_metrics_registry() -> &'static Registry {
     REGISTRY.get_or_init(|| prometheus::default_registry().clone())
 }
 
-pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
+// TODO(MrCroxx): Use `LazyLock` after `lazy_cell` is stable.
+// pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
+
+lazy_static::lazy_static! {
+    pub static ref METRICS: GlobalMetrics = GlobalMetrics::default();
+}
 
 #[derive(Debug)]
 pub struct GlobalMetrics {
diff --git a/foyer-experimental/src/wal.rs b/foyer-experimental/src/wal.rs
index 517e63b6..0d63b4e4 100644
--- a/foyer-experimental/src/wal.rs
+++ b/foyer-experimental/src/wal.rs
@@ -25,15 +25,11 @@ use std::{
 
 use bytes::{Buf, BufMut};
 use crossbeam::channel;
+use foyer_common::code::{BufExt, BufMutExt};
 use parking_lot::{Condvar, Mutex};
 use tokio::sync::oneshot;
 
-use crate::{
-    asyncify,
-    buf::{BufExt, BufMutExt},
-    error::Result,
-    metrics::Metrics,
-};
+use crate::{asyncify, error::Result, metrics::Metrics};
 
 pub trait HashValue: Send + Sync + 'static + Eq + std::fmt::Debug {
     fn size() -> usize;
diff --git a/foyer-intrusive/src/eviction/mod.rs b/foyer-intrusive/src/eviction/mod.rs
index 614e1db6..263e24a9 100644
--- a/foyer-intrusive/src/eviction/mod.rs
+++ b/foyer-intrusive/src/eviction/mod.rs
@@ -16,11 +16,9 @@ use std::fmt::Debug;
 
 use crate::core::adapter::Adapter;
 
-pub trait Config = Send + Sync + 'static + Debug + Clone;
-
 pub trait EvictionPolicy: Send + Sync + Debug + 'static {
     type Adapter: Adapter;
-    type Config: Config;
+    type Config: Send + Sync + 'static + Debug + Clone;
 
     fn new(config: Self::Config) -> Self;
 
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 6d6fbc28..038d1c2c 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -12,8 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(ptr_metadata)]
-#![feature(trait_alias)]
 #![feature(offset_of)]
 // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
 #![allow(clippy::new_without_default)]
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index b6069b8e..4aa1bb87 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -22,6 +22,7 @@ foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
 foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.12"
+lazy_static = "1"
 libc = "0.2"
 lz4 = "1.24"
 memoffset = "0.9"
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 8090ef7e..0f600b65 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -350,7 +350,7 @@ mod tests {
 
             let res = buffer.write(entry).await;
             let entry = match res {
-                Err(BufferError::NeedRotate(entry)) => Box::into_inner(entry),
+                Err(BufferError::NeedRotate(entry)) => *entry,
                 _ => panic!("should be not enough error"),
             };
 
@@ -397,7 +397,7 @@ mod tests {
 
             let res = buffer.write(entry).await;
             let entry = match res {
-                Err(BufferError::NeedRotate(entry)) => Box::into_inner(entry),
+                Err(BufferError::NeedRotate(entry)) => *entry,
                 _ => panic!("should be not enough error"),
             };
 
diff --git a/foyer-storage/src/device/allocator.rs b/foyer-storage/src/device/allocator.rs
index c848417b..3919334c 100644
--- a/foyer-storage/src/device/allocator.rs
+++ b/foyer-storage/src/device/allocator.rs
@@ -52,14 +52,14 @@ mod tests {
         let allocator = AlignedAllocator::new(ALIGN);
 
         let mut buf: Vec<u8, _> = Vec::with_capacity_in(ALIGN * 8, &allocator);
-        bits::assert_aligned(ALIGN, buf.as_ptr().addr());
+        bits::assert_aligned(ALIGN, buf.as_ptr() as _);
 
         buf.extend_from_slice(&[b'x'; ALIGN * 8]);
-        bits::assert_aligned(ALIGN, buf.as_ptr().addr());
+        bits::assert_aligned(ALIGN, buf.as_ptr() as _);
         assert_eq!(buf, [b'x'; ALIGN * 8]);
 
         buf.extend_from_slice(&[b'x'; ALIGN * 8]);
-        bits::assert_aligned(ALIGN, buf.as_ptr().addr());
+        bits::assert_aligned(ALIGN, buf.as_ptr() as _);
         assert_eq!(buf, [b'x'; ALIGN * 16])
     }
 }
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index fc52bc38..1cae8ba6 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -24,10 +24,21 @@ use futures::Future;
 
 use crate::region::RegionId;
 
-pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
-pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
-pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
-pub trait IoRange = RangeBoundsExt<usize> + Sized + Send + Sync + 'static + Debug;
+// TODO(MrCroxx): Use `trait_alias` after stable.
+
+// pub trait BufferAllocator = Allocator + Clone + Send + Sync + 'static + Debug;
+// pub trait IoBuf = AsRef<[u8]> + Send + Sync + 'static + Debug;
+// pub trait IoBufMut = AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug;
+// pub trait IoRange = RangeBoundsExt<usize> + Sized + Send + Sync + 'static + Debug;
+
+pub trait BufferAllocator: Allocator + Clone + Send + Sync + 'static + Debug {}
+impl<T: Allocator + Clone + Send + Sync + 'static + Debug> BufferAllocator for T {}
+pub trait IoBuf: AsRef<[u8]> + Send + Sync + 'static + Debug {}
+impl<T: AsRef<[u8]> + Send + Sync + 'static + Debug> IoBuf for T {}
+pub trait IoBufMut: AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug {}
+impl<T: AsRef<[u8]> + AsMut<[u8]> + Send + Sync + 'static + Debug> IoBufMut for T {}
+pub trait IoRange: RangeBoundsExt<usize> + Sized + Send + Sync + 'static + Debug {}
+impl<T: RangeBoundsExt<usize> + Sized + Send + Sync + 'static + Debug> IoRange for T {}
 
 pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
     type IoBufferAllocator: BufferAllocator;
diff --git a/foyer-storage/src/flusher.rs b/foyer-storage/src/flusher.rs
index 6b19a329..84697268 100644
--- a/foyer-storage/src/flusher.rs
+++ b/foyer-storage/src/flusher.rs
@@ -144,7 +144,7 @@ where
         let old_region = self.buffer.region();
 
         let entry = match self.buffer.write(entry).await {
-            Err(BufferError::NeedRotate(entry)) => Box::into_inner(entry),
+            Err(BufferError::NeedRotate(entry)) => *entry,
             Ok(entries) => return self.update_catalog(entries).await,
             Err(e) => return Err(e.into()),
         };
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 7e2cf9a1..eadf2d09 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -13,13 +13,6 @@
 //  limitations under the License.
 
 #![feature(allocator_api)]
-#![feature(strict_provenance)]
-#![feature(trait_alias)]
-#![feature(get_mut_unchecked)]
-#![feature(error_generic_member_access)]
-#![feature(lazy_cell)]
-#![feature(box_into_inner)]
-#![feature(try_trait_v2)]
 #![feature(offset_of)]
 
 pub mod admission;
diff --git a/foyer-storage/src/metrics.rs b/foyer-storage/src/metrics.rs
index 2512fc2c..a8334435 100644
--- a/foyer-storage/src/metrics.rs
+++ b/foyer-storage/src/metrics.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::sync::{LazyLock, OnceLock};
+use std::sync::OnceLock;
 
 use prometheus::{
     core::{AtomicU64, GenericGauge, GenericGaugeVec},
@@ -54,8 +54,13 @@ pub fn get_metrics_registry() -> &'static Registry {
     REGISTRY.get_or_init(|| prometheus::default_registry().clone())
 }
 
-/// Multiple foyer instance will share the same global metrics with different label `foyer` name.
-pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
+// TODO(MrCroxx): Use `LazyLock` after `lazy_cell` is stable.
+// /// Multiple foyer instance will share the same global metrics with different label `foyer` name.
+// pub static METRICS: LazyLock<GlobalMetrics> = LazyLock::new(GlobalMetrics::default);
+
+lazy_static::lazy_static! {
+    pub static ref METRICS: GlobalMetrics = GlobalMetrics::default();
+}
 
 #[derive(Debug)]
 pub struct GlobalMetrics {
diff --git a/foyer-storage/src/storage.rs b/foyer-storage/src/storage.rs
index 3909731a..0e9a70d6 100644
--- a/foyer-storage/src/storage.rs
+++ b/foyer-storage/src/storage.rs
@@ -19,7 +19,11 @@ use futures::Future;
 
 use crate::{compress::Compression, error::Result};
 
-pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
+// TODO(MrCroxx): Use `trait_alias` after stable.
+// pub trait FetchValueFuture<V> = Future<Output = anyhow::Result<V>> + Send + 'static;
+
+pub trait FetchValueFuture<V>: Future<Output = anyhow::Result<V>> + Send + 'static {}
+impl<V, T: Future<Output = anyhow::Result<V>> + Send + 'static> FetchValueFuture<V> for T {}
 
 pub trait StorageWriter: Send + Sync + Debug {
     type Key: Key;
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 191d0932..32d97e4d 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -12,9 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(trait_alias)]
-#![feature(pattern)]
-
 pub use foyer_common as common;
 pub use foyer_intrusive as intrusive;
 pub use foyer_memory as memory;

From e423dc46a7e063ee76941e8996f36cb5142725c5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 10 Apr 2024 18:09:18 +0800
Subject: [PATCH 246/261] refactor: use crate allocator_api2 to make foyer
 build on stable (#317)

* refactor: use crate allocator_api2 to make foyer build on stable

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: use stable pipeline for ci tests, use nightly for sanitizer

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: update ci prefix key

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: try fix ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: update ci nightly version

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: try resolve github issue

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml         | 17 +++++-----
 .github/workflows/main.yml            | 17 +++++-----
 .github/workflows/pull-request.yml    | 17 +++++-----
 Makefile                              |  6 ++--
 foyer-common/src/lib.rs               |  1 -
 foyer-experimental/src/wal.rs         |  7 +++-
 foyer-intrusive/src/core/adapter.rs   |  6 ----
 foyer-intrusive/src/lib.rs            |  3 --
 foyer-memory/src/lib.rs               |  2 --
 foyer-storage/Cargo.toml              |  1 +
 foyer-storage/src/buffer.rs           | 15 +++++----
 foyer-storage/src/device/allocator.rs | 46 ++++++++++++++++++++++++---
 foyer-storage/src/device/fs.rs        |  5 +--
 foyer-storage/src/device/mod.rs       | 11 ++++---
 foyer-storage/src/lib.rs              |  3 --
 foyer-storage/src/region.rs           |  7 ++--
 foyer-workspace-hack/Cargo.toml       |  1 +
 rust-toolchain                        |  2 --
 rustfmt.toml                          | 10 +++---
 19 files changed, 109 insertions(+), 68 deletions(-)
 delete mode 100644 rust-toolchain

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 6b448a05..3c3ffc11 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -3,9 +3,10 @@ name:
 on:
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-12-26
+  RUST_TOOLCHAIN: stable
+  RUST_TOOLCHAIN_NIGHTLY: nightly-2024-03-17
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20240306
+  CACHE_KEY_SUFFIX: 20240410-2
 
 jobs:
   misc-check:
@@ -132,7 +133,7 @@ jobs:
       - name: Install rust toolchain@v1
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
@@ -149,14 +150,14 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   lsan:
@@ -168,7 +169,7 @@ jobs:
       - name: Install rust toolchain@v1
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
@@ -185,14 +186,14 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Leak Sanitizer
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a80dcb10..be6d379a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,9 +11,10 @@ on:
     branches: [main]
   workflow_dispatch:
 env:
-  RUST_TOOLCHAIN: nightly-2023-12-26
+  RUST_TOOLCHAIN: stable
+  RUST_TOOLCHAIN_NIGHTLY: nightly-2024-03-17
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20240306
+  CACHE_KEY_SUFFIX: 20240410-2
 jobs:
   misc-check:
     name: misc check
@@ -139,7 +140,7 @@ jobs:
       - name: Install rust toolchain@v1
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
@@ -156,14 +157,14 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   lsan:
@@ -175,7 +176,7 @@ jobs:
       - name: Install rust toolchain@v1
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
@@ -192,14 +193,14 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Leak Sanitizer
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 75518119..0133faa8 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -10,9 +10,10 @@ on:
   pull_request:
     branches: [main]
 env:
-  RUST_TOOLCHAIN: nightly-2023-12-26
+  RUST_TOOLCHAIN: stable
+  RUST_TOOLCHAIN_NIGHTLY: nightly-2024-03-17
   CARGO_TERM_COLOR: always
-  CACHE_KEY_SUFFIX: 20240306
+  CACHE_KEY_SUFFIX: 20240410-2
 jobs:
   misc-check:
     name: misc check
@@ -138,7 +139,7 @@ jobs:
       - name: Install rust toolchain@v1
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
@@ -155,14 +156,14 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Address Sanitizer
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   lsan:
@@ -174,7 +175,7 @@ jobs:
       - name: Install rust toolchain@v1
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
       - name: Cache Cargo home
         uses: actions/cache@v2
         id: cache
@@ -191,14 +192,14 @@ jobs:
           RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
       - name: Run foyer-storage-bench With Leak Sanitizer
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
           RUST_LOG: info
         run: |-
-          cargo build --all --target x86_64-unknown-linux-gnu
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
   deterministic-test:
diff --git a/Makefile b/Makefile
index fd9cfa87..72c61773 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,8 @@ check:
 	cargo sort -w
 	cargo fmt --all
 	cargo clippy --all-targets
-	cargo udeps --workspace --exclude foyer-workspace-hack
+	# TODO(MrCroxx): Restore udeps check after it doesn't requires nightly anymore.
+	# cargo udeps --workspace --exclude foyer-workspace-hack
 
 check-all:
 	shellcheck ./scripts/*
@@ -28,7 +29,8 @@ check-all:
 	cargo clippy --all-targets --features tokio-console
 	cargo clippy --all-targets --features trace
 	cargo clippy --all-targets
-	cargo udeps --workspace --exclude foyer-workspace-hack
+	# TODO(MrCroxx): Restore udeps check after it doesn't requires nightly anymore.
+	# cargo udeps --workspace --exclude foyer-workspace-hack
 
 test:
 	RUST_BACKTRACE=1 cargo nextest run --all
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 735c635e..adb30f00 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -12,7 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(bound_map)]
 #![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 
 pub mod async_queue;
diff --git a/foyer-experimental/src/wal.rs b/foyer-experimental/src/wal.rs
index 0d63b4e4..c5df56d6 100644
--- a/foyer-experimental/src/wal.rs
+++ b/foyer-experimental/src/wal.rs
@@ -143,7 +143,12 @@ impl<H: HashValue> TombstoneLog<H> {
 
         path.push(format!("tombstone-{:08X}", config.id));
 
-        let file = OpenOptions::new().write(true).read(true).create(true).open(path)?;
+        let file = OpenOptions::new()
+            .write(true)
+            .read(true)
+            .create(true)
+            .truncate(true)
+            .open(path)?;
 
         let inner = Arc::new(TombstoneLogInner {
             inflights: Mutex::new(vec![]),
diff --git a/foyer-intrusive/src/core/adapter.rs b/foyer-intrusive/src/core/adapter.rs
index 3f98d74e..eb09afbd 100644
--- a/foyer-intrusive/src/core/adapter.rs
+++ b/foyer-intrusive/src/core/adapter.rs
@@ -117,8 +117,6 @@ pub unsafe trait PriorityAdapter: Adapter {
 /// # Examples
 ///
 /// ```
-/// #![feature(offset_of)]
-///
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
 /// use foyer_intrusive::core::pointer::Pointer;
@@ -215,8 +213,6 @@ macro_rules! intrusive_adapter {
 /// # Examples
 ///
 /// ```
-/// #![feature(offset_of)]
-///
 /// use foyer_intrusive::{intrusive_adapter, key_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, KeyAdapter, Link};
 /// use foyer_intrusive::core::pointer::Pointer;
@@ -286,8 +282,6 @@ macro_rules! key_adapter {
 /// # Examples
 ///
 /// ```
-/// #![feature(offset_of)]
-///
 /// use foyer_intrusive::{intrusive_adapter, priority_adapter};
 /// use foyer_intrusive::core::adapter::{Adapter, PriorityAdapter, Link};
 /// use foyer_intrusive::core::pointer::Pointer;
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 038d1c2c..5fde39b0 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -12,7 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(offset_of)]
 // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
 #![allow(clippy::new_without_default)]
 
@@ -24,8 +23,6 @@ pub use memoffset::offset_of;
 /// # Examples
 ///
 /// ```
-/// #![feature(offset_of)]
-///
 /// use foyer_intrusive::container_of;
 ///
 /// struct S { x: u32, y: u32 };
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index 3b92ad5a..fe0e8ae6 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -62,8 +62,6 @@
 //! The handle that does not appear in either the indexer or the eviction container, and has no external owner, will be
 //! destroyed.
 
-#![feature(offset_of)]
-
 pub trait Key: Send + Sync + 'static + std::hash::Hash + Eq + Ord {}
 pub trait Value: Send + Sync + 'static {}
 
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 4aa1bb87..d15e0a33 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -13,6 +13,7 @@ homepage = "https://github.com/mrcroxx/foyer"
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
+allocator-api2 = "0.2" # TODO(MrCroxx): Remove this after `allocator_api` is stable.
 anyhow = "1.0"
 bitflags = "2.3.1"
 bitmaps = "3.2"
diff --git a/foyer-storage/src/buffer.rs b/foyer-storage/src/buffer.rs
index 0f600b65..5f5f2fe1 100644
--- a/foyer-storage/src/buffer.rs
+++ b/foyer-storage/src/buffer.rs
@@ -14,6 +14,7 @@
 
 use std::fmt::Debug;
 
+use allocator_api2::vec::Vec as VecA;
 use foyer_common::{
     bits::{align_up, is_aligned},
     code::{Cursor, Key, Value},
@@ -21,7 +22,7 @@ use foyer_common::{
 
 use crate::{
     compress::Compression,
-    device::{error::DeviceError, Device},
+    device::{allocator::WritableVecA, error::DeviceError, Device},
     flusher::Entry,
     generic::{checksum, EntryHeader},
     region::{RegionHeader, RegionId, Version, REGION_MAGIC},
@@ -63,7 +64,7 @@ where
 {
     // TODO(MrCroxx): optimize buffer allocation
     /// io buffer
-    buffer: Vec<u8, D::IoBufferAllocator>,
+    buffer: VecA<u8, D::IoBufferAllocator>,
 
     /// current writing region
     region: Option<RegionId>,
@@ -227,15 +228,17 @@ where
         let mut vcursor = value.into_cursor();
         match compression {
             Compression::None => {
-                std::io::copy(&mut vcursor, &mut self.buffer).map_err(DeviceError::from)?;
+                std::io::copy(&mut vcursor, &mut WritableVecA(&mut self.buffer)).map_err(DeviceError::from)?;
             }
             Compression::Zstd => {
-                zstd::stream::copy_encode(&mut vcursor, &mut self.buffer, 0).map_err(DeviceError::from)?;
+                zstd::stream::copy_encode(&mut vcursor, &mut WritableVecA(&mut self.buffer), 0)
+                    .map_err(DeviceError::from)?;
             }
             Compression::Lz4 => {
+                let buf = &mut WritableVecA(&mut self.buffer);
                 let mut encoder = lz4::EncoderBuilder::new()
                     .checksum(lz4::ContentChecksum::NoChecksum)
-                    .build(&mut self.buffer)
+                    .build(buf)
                     .map_err(DeviceError::from)?;
                 std::io::copy(&mut vcursor, &mut encoder).map_err(DeviceError::from)?;
                 let (_w, res) = encoder.finish();
@@ -247,7 +250,7 @@ where
 
         // write key
         let mut kcursor = key.into_cursor();
-        std::io::copy(&mut kcursor, &mut self.buffer).map_err(DeviceError::from)?;
+        std::io::copy(&mut kcursor, &mut WritableVecA(&mut self.buffer)).map_err(DeviceError::from)?;
         let encoded_key_len = self.buffer.len() - cursor;
         cursor = self.buffer.len();
 
diff --git a/foyer-storage/src/device/allocator.rs b/foyer-storage/src/device/allocator.rs
index 3919334c..81091f23 100644
--- a/foyer-storage/src/device/allocator.rs
+++ b/foyer-storage/src/device/allocator.rs
@@ -12,10 +12,48 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::alloc::{Allocator, Global};
-
+use allocator_api2::{
+    alloc::{AllocError, Allocator, Global},
+    vec::Vec as VecA,
+};
 use foyer_common::bits;
 
+pub struct WritableVecA<'a, T, A: Allocator>(pub &'a mut VecA<T, A>);
+
+impl<'a, A: Allocator> std::io::Write for WritableVecA<'a, u8, A> {
+    #[inline]
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.0.extend_from_slice(buf);
+        Ok(buf.len())
+    }
+
+    #[inline]
+    fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result<usize> {
+        let len = bufs.iter().map(|b| b.len()).sum();
+        self.0.reserve(len);
+        for buf in bufs {
+            self.0.extend_from_slice(buf);
+        }
+        Ok(len)
+    }
+
+    // #[inline]
+    // fn is_write_vectored(&self) -> bool {
+    //     true
+    // }
+
+    #[inline]
+    fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> {
+        self.0.extend_from_slice(buf);
+        Ok(())
+    }
+
+    #[inline]
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct AlignedAllocator {
     align: usize,
@@ -29,7 +67,7 @@ impl AlignedAllocator {
 }
 
 unsafe impl Allocator for AlignedAllocator {
-    fn allocate(&self, layout: std::alloc::Layout) -> Result<std::ptr::NonNull<[u8]>, std::alloc::AllocError> {
+    fn allocate(&self, layout: std::alloc::Layout) -> Result<std::ptr::NonNull<[u8]>, AllocError> {
         let layout =
             std::alloc::Layout::from_size_align(layout.size(), bits::align_up(self.align, layout.align())).unwrap();
         Global.allocate(layout)
@@ -51,7 +89,7 @@ mod tests {
         const ALIGN: usize = 512;
         let allocator = AlignedAllocator::new(ALIGN);
 
-        let mut buf: Vec<u8, _> = Vec::with_capacity_in(ALIGN * 8, &allocator);
+        let mut buf: VecA<u8, _> = VecA::with_capacity_in(ALIGN * 8, &allocator);
         bits::assert_aligned(ALIGN, buf.as_ptr() as _);
 
         buf.extend_from_slice(&[b'x'; ALIGN * 8]);
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index d7ba50d2..b03fbc12 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -19,6 +19,7 @@ use std::{
     sync::Arc,
 };
 
+use allocator_api2::vec::Vec as VecA;
 use foyer_common::range::RangeBoundsExt;
 use futures::future::try_join_all;
 use itertools::Itertools;
@@ -172,9 +173,9 @@ impl Device for FsDevice {
         &self.inner.io_buffer_allocator
     }
 
-    fn io_buffer(&self, len: usize, capacity: usize) -> Vec<u8, Self::IoBufferAllocator> {
+    fn io_buffer(&self, len: usize, capacity: usize) -> VecA<u8, Self::IoBufferAllocator> {
         assert!(len <= capacity);
-        let mut buf = Vec::with_capacity_in(capacity, self.inner.io_buffer_allocator);
+        let mut buf = VecA::with_capacity_in(capacity, self.inner.io_buffer_allocator);
         unsafe { buf.set_len(len) };
         buf
     }
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 1cae8ba6..2e35ccfc 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -16,8 +16,9 @@ pub mod allocator;
 pub mod error;
 pub mod fs;
 
-use std::{alloc::Allocator, fmt::Debug};
+use std::fmt::Debug;
 
+use allocator_api2::{alloc::Allocator, vec::Vec as VecA};
 use error::DeviceResult;
 use foyer_common::range::RangeBoundsExt;
 use futures::Future;
@@ -84,7 +85,7 @@ pub trait Device: Sized + Clone + Send + Sync + 'static + Debug {
 
     fn io_buffer_allocator(&self) -> &Self::IoBufferAllocator;
 
-    fn io_buffer(&self, len: usize, capacity: usize) -> Vec<u8, Self::IoBufferAllocator>;
+    fn io_buffer(&self, len: usize, capacity: usize) -> VecA<u8, Self::IoBufferAllocator>;
 
     fn region_size(&self) -> usize {
         debug_assert!(self.capacity() % self.regions() == 0);
@@ -98,7 +99,7 @@ pub trait DeviceExt: Device {
         &self,
         region: RegionId,
         range: impl IoRange,
-    ) -> impl Future<Output = DeviceResult<Vec<u8, Self::IoBufferAllocator>>> + Send {
+    ) -> impl Future<Output = DeviceResult<VecA<u8, Self::IoBufferAllocator>>> + Send {
         async move {
             let range = range.bounds(0..self.region_size());
             let size = range.size().unwrap();
@@ -218,8 +219,8 @@ pub mod tests {
             &self.0
         }
 
-        fn io_buffer(&self, len: usize, capacity: usize) -> Vec<u8, Self::IoBufferAllocator> {
-            let mut buf = Vec::with_capacity_in(capacity, self.0);
+        fn io_buffer(&self, len: usize, capacity: usize) -> VecA<u8, Self::IoBufferAllocator> {
+            let mut buf = VecA::with_capacity_in(capacity, self.0);
             unsafe { buf.set_len(len) };
             buf
         }
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index eadf2d09..3c54019b 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -12,9 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![feature(allocator_api)]
-#![feature(offset_of)]
-
 pub mod admission;
 pub mod buffer;
 pub mod catalog;
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 4280d395..ef98e1d2 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -22,6 +22,7 @@ use std::{
     },
 };
 
+use allocator_api2::vec::Vec as VecA;
 use bytes::{Buf, BufMut};
 use foyer_common::range::RangeBoundsExt;
 use parking_lot::Mutex;
@@ -107,7 +108,7 @@ where
 {
     // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
     #[allow(clippy::type_complexity)]
-    waits: BTreeMap<(usize, usize), Vec<oneshot::Sender<Result<Arc<Vec<u8, A>>>>>>,
+    waits: BTreeMap<(usize, usize), Vec<oneshot::Sender<Result<Arc<VecA<u8, A>>>>>>,
 }
 
 #[derive(Debug, Clone)]
@@ -156,7 +157,7 @@ where
     // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
     #[allow(clippy::type_complexity)]
     #[tracing::instrument(skip(self, view))]
-    pub async fn load(&self, view: RegionView) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
+    pub async fn load(&self, view: RegionView) -> Result<Option<Arc<VecA<u8, D::IoBufferAllocator>>>> {
         let res = self
             .load_range(view.offset as usize..view.offset as usize + view.len as usize)
             .await;
@@ -172,7 +173,7 @@ where
     pub async fn load_range(
         &self,
         range: impl RangeBounds<usize>,
-    ) -> Result<Option<Arc<Vec<u8, D::IoBufferAllocator>>>> {
+    ) -> Result<Option<Arc<VecA<u8, D::IoBufferAllocator>>>> {
         let range = range.bounds(0..self.device.region_size());
 
         let rx = {
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 153f7282..d18a747c 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -18,6 +18,7 @@ publish = true
 ### BEGIN HAKARI SECTION
 [dependencies]
 ahash = { version = "0.8" }
+allocator-api2 = { version = "0.2" }
 crossbeam-channel = { version = "0.5" }
 crossbeam-epoch = { version = "0.9" }
 crossbeam-utils = { version = "0.8" }
diff --git a/rust-toolchain b/rust-toolchain
deleted file mode 100644
index ef0ed508..00000000
--- a/rust-toolchain
+++ /dev/null
@@ -1,2 +0,0 @@
-[toolchain]
-channel = "nightly-2023-12-26"
\ No newline at end of file
diff --git a/rustfmt.toml b/rustfmt.toml
index d9905931..16bafd46 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1,6 +1,8 @@
-imports_granularity = "Crate"
-group_imports = "StdExternalCrate"
+# TODO(MrCroxx): Restore comments after the features are stable.
+
+# imports_granularity = "Crate"
+# group_imports = "StdExternalCrate"
 tab_spaces = 4
-wrap_comments = true
+# wrap_comments = true
 max_width = 120
-comment_width = 120
\ No newline at end of file
+# comment_width = 120

From 3e8f334b23c7535f614eb4922bbd2c84251ea7da Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 11 Apr 2024 00:32:16 +0800
Subject: [PATCH 247/261] test: run CI on both linux and macos (#318)

* test: run CI on both linux and macos

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix build on macos

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: try fix ci run on targets

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml         | 5 ++++-
 .github/workflows/main.yml            | 5 ++++-
 .github/workflows/pull-request.yml    | 5 ++++-
 foyer-experimental-bench/src/utils.rs | 6 ++++--
 foyer-storage-bench/src/utils.rs      | 6 ++++--
 foyer-storage/src/device/fs.rs        | 3 ++-
 6 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 3c3ffc11..5b368c44 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -36,7 +36,10 @@ jobs:
         uses: ludeeus/action-shellcheck@master
   rust-test:
     name: rust test with codecov
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index be6d379a..8107c45d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -43,7 +43,10 @@ jobs:
         uses: ludeeus/action-shellcheck@master
   rust-test:
     name: rust test with codecov
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 0133faa8..5d813c93 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -42,7 +42,10 @@ jobs:
         uses: ludeeus/action-shellcheck@master
   rust-test:
     name: rust test with codecov
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
diff --git a/foyer-experimental-bench/src/utils.rs b/foyer-experimental-bench/src/utils.rs
index f640b3c9..17a5f7e2 100644
--- a/foyer-experimental-bench/src/utils.rs
+++ b/foyer-experimental-bench/src/utils.rs
@@ -31,7 +31,8 @@ use std::path::{Path, PathBuf};
 use itertools::Itertools;
 use nix::{fcntl::readlink, sys::stat::stat};
 
-#[cfg_attr(not(target_os = "linux"), expect(dead_code))]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
 #[derive(PartialEq, Clone, Copy, Debug)]
 pub enum FsType {
     Xfs,
@@ -41,7 +42,8 @@ pub enum FsType {
     Others,
 }
 
-#[cfg_attr(not(target_os = "linux"), expect(unused_variables))]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[cfg_attr(not(target_os = "linux"), allow(unused_variables))]
 pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
     #[cfg(target_os = "linux")]
     {
diff --git a/foyer-storage-bench/src/utils.rs b/foyer-storage-bench/src/utils.rs
index c2a40a0f..cf703186 100644
--- a/foyer-storage-bench/src/utils.rs
+++ b/foyer-storage-bench/src/utils.rs
@@ -31,7 +31,8 @@ use std::path::{Path, PathBuf};
 use itertools::Itertools;
 use nix::{fcntl::readlink, sys::stat::stat};
 
-#[cfg_attr(not(target_os = "linux"), expect(dead_code))]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
 #[derive(PartialEq, Clone, Copy, Debug)]
 pub enum FsType {
     Xfs,
@@ -41,7 +42,8 @@ pub enum FsType {
     Others,
 }
 
-#[cfg_attr(not(target_os = "linux"), expect(unused_variables))]
+// TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+#[cfg_attr(not(target_os = "linux"), allow(unused_variables))]
 pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
     #[cfg(target_os = "linux")]
     {
diff --git a/foyer-storage/src/device/fs.rs b/foyer-storage/src/device/fs.rs
index b03fbc12..06ef1eb5 100644
--- a/foyer-storage/src/device/fs.rs
+++ b/foyer-storage/src/device/fs.rs
@@ -62,7 +62,8 @@ impl FsDeviceConfig {
 struct FsDeviceInner {
     config: FsDeviceConfig,
 
-    #[cfg_attr(not(target_os = "linux"), expect(dead_code))]
+    // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
     dir: File,
 
     files: Vec<File>,

From 475aee046b1051b0ec9dee400e39559b0613f2b3 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 11 Apr 2024 14:27:41 +0800
Subject: [PATCH 248/261] doc: update README and add a simple example (#319)

* doc: update README and add a simple example

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update Cargo.toml

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .config/hakari.toml                 |  1 +
 Cargo.toml                          |  1 +
 README.md                           | 29 ++++++++++++++++++++----
 examples/Cargo.toml                 | 19 ++++++++++++++++
 examples/memory.rs                  | 35 +++++++++++++++++++++++++++++
 foyer-common/Cargo.toml             |  1 +
 foyer-experimental-bench/Cargo.toml |  1 +
 foyer-experimental/Cargo.toml       |  1 +
 foyer-intrusive/Cargo.toml          |  1 +
 foyer-memory/Cargo.toml             |  1 +
 foyer-storage-bench/Cargo.toml      |  1 +
 foyer-storage/Cargo.toml            |  1 +
 foyer-workspace-hack/Cargo.toml     |  1 +
 foyer/Cargo.toml                    |  1 +
 14 files changed, 90 insertions(+), 4 deletions(-)
 create mode 100644 examples/Cargo.toml
 create mode 100644 examples/memory.rs

diff --git a/.config/hakari.toml b/.config/hakari.toml
index f3329bba..e279752e 100644
--- a/.config/hakari.toml
+++ b/.config/hakari.toml
@@ -24,6 +24,7 @@ platforms = [
 # exact-versions = true
 
 [traversal-excludes]
+workspace-members = ["examples"]
 third-party = [
     { name = "miniz_oxide" },
 ]
\ No newline at end of file
diff --git a/Cargo.toml b/Cargo.toml
index e91dfe93..e11d5e22 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
+    "examples",
     "foyer",
     "foyer-common",
     "foyer-experimental",
diff --git a/README.md b/README.md
index 67397938..7e15e3f8 100644
--- a/README.md
+++ b/README.md
@@ -6,16 +6,37 @@
 
 *foyer* is inspired by [Facebook/CacheLib](https://github.com/facebook/cachelib), which is an excellent hybrid cache lib in C++. *foyer* is not only a 'rewrite in Rust project', but provide some features that *CacheLib* doesn't have for now.
 
+## Supported Rust Versions
+
+*foyer* is built against the latest stable release. The minimum supported version is 1.77.2. The current *foyer* version is not guaranteed to build on Rust versions earlier than the minimum supported version.
+
 ## Development state
 
 Currently, *foyer* only finished few features, and is still under heavy development.
 
+## Features
+
+- [x] in-memory cache
+  - [x] FIFO
+  - [x] LRU with priority pool
+  - [x] 3-qeue w-TinyLFU (imspired by [caffeine](https://github.com/ben-manes/caffeine))
+  - [x] S3FIFO without Ghost Queue
+- [x] disk cache
+- [ ] TTL (time to live)
+
+## Examples
+
+The examples can be found [here](https://github.com/MrCroxx/foyer/tree/main/examples).
+
 ## Roadmap
 
-- [ ] Better intrusive index collectios for memory cache.
-- [ ] Hybrid memory and disk cache.
-- [ ] Raw device and single file device support.
-- [ ] More detailed metrics or statistics.
+- [ ] More user-friendly API.
+- [ ] User-friendly Documents and examples.
+- [ ] Support TTL.
+- [ ] Simplify `foyer-storage`.
+- [ ] Refactor `foyer-storage` region reclaiming policy.
+- [ ] Support on Windows.
+- [ ] Unify in-memory cache and disk cache configuration.
 
 ## Contributing
 
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
new file mode 100644
index 00000000..b7f82eb6
--- /dev/null
+++ b/examples/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "examples"
+version = "0.0.0"
+edition = "2021"
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+description = "Hybrid cache for Rust"
+license = "Apache-2.0"
+repository = "https://github.com/mrcroxx/foyer"
+homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+publish = false
+
+[dependencies]
+foyer = { version = "*", path = "../foyer" }
+
+[[example]]
+name = "memory"
+path = "memory.rs"
diff --git a/examples/memory.rs b/examples/memory.rs
new file mode 100644
index 00000000..db8be486
--- /dev/null
+++ b/examples/memory.rs
@@ -0,0 +1,35 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::hash::RandomState;
+
+use foyer::memory::{Cache, DefaultCacheEventListener, LruCacheConfig, LruConfig};
+
+fn main() {
+    let cache = Cache::lru(LruCacheConfig {
+        capacity: 16,
+        shards: 4,
+        eviction_config: LruConfig {
+            high_priority_pool_ratio: 0.1,
+        },
+        object_pool_capacity: 1024,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    });
+
+    let entry = cache.insert("hello".to_string(), "world".to_string(), 1);
+    let e = cache.get(&"hello".to_string()).unwrap();
+
+    assert_eq!(entry.value(), e.value());
+}
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 0af9293f..0abd488a 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -7,6 +7,7 @@ description = "common utils for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-experimental-bench/Cargo.toml b/foyer-experimental-bench/Cargo.toml
index 6d368527..4552d34d 100644
--- a/foyer-experimental-bench/Cargo.toml
+++ b/foyer-experimental-bench/Cargo.toml
@@ -7,6 +7,7 @@ description = "storage engine bench tool for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 autobenches = false
 
diff --git a/foyer-experimental/Cargo.toml b/foyer-experimental/Cargo.toml
index 48bdc161..ab8b0a50 100644
--- a/foyer-experimental/Cargo.toml
+++ b/foyer-experimental/Cargo.toml
@@ -7,6 +7,7 @@ description = "experimental components for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index f1356e72..8b33072a 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -7,6 +7,7 @@ description = "intrusive data structures for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 2d151b69..9c26a60c 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -7,6 +7,7 @@ description = "memory cache for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index 83bdbf01..e8400682 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -7,6 +7,7 @@ description = "storage engine bench tool for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index d15e0a33..4eb2174a 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -7,6 +7,7 @@ description = "storage engine for foyer - the hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index d18a747c..03c4afc0 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -10,6 +10,7 @@ description = "workspace-hack package, managed by hakari"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.
 publish = true
 # The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index ec556e48..be3959cf 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -7,6 +7,7 @@ description = "Hybrid cache for Rust"
 license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
+readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]

From 2abe3df5fa917c3a7915f802810a53099961554f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 11 Apr 2024 15:32:57 +0800
Subject: [PATCH 249/261] chore: add msrv for foyer, update bager in readme
 (#321)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 README.md        | 8 +++++++-
 foyer/Cargo.toml | 1 +
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7e15e3f8..ff259e56 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,12 @@
 # foyer
 
-[![CI (main)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml) [![License Checker](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml) [![codecov](https://codecov.io/github/MrCroxx/foyer/branch/main/graph/badge.svg?token=YO33YQCB70)](https://codecov.io/github/MrCroxx/foyer)
+![Crates.io Version](https://img.shields.io/crates/v/foyer)
+![Crates.io MSRV](https://img.shields.io/crates/msrv/foyer)
+![GitHub License](https://img.shields.io/github/license/mrcroxx/foyer)
+
+[![CI (main)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/main.yml)
+[![License Checker](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml/badge.svg)](https://github.com/MrCroxx/foyer/actions/workflows/license_check.yml)
+[![codecov](https://codecov.io/github/MrCroxx/foyer/branch/main/graph/badge.svg?token=YO33YQCB70)](https://codecov.io/github/MrCroxx/foyer)
 
 *foyer* aims to be a user-friendly hybrid cache lib in Rust. 
 
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index be3959cf..4c9fb883 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
 readme = "../README.md"
+rust-version = "1.77.2"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]

From cb2616e9d81b4849a3f66012525a798e5814a075 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 11 Apr 2024 15:55:18 +0800
Subject: [PATCH 250/261] fix: do not truncate when open wal (#322)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-experimental/src/wal.rs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/foyer-experimental/src/wal.rs b/foyer-experimental/src/wal.rs
index c5df56d6..7d2673f5 100644
--- a/foyer-experimental/src/wal.rs
+++ b/foyer-experimental/src/wal.rs
@@ -143,12 +143,9 @@ impl<H: HashValue> TombstoneLog<H> {
 
         path.push(format!("tombstone-{:08X}", config.id));
 
-        let file = OpenOptions::new()
-            .write(true)
-            .read(true)
-            .create(true)
-            .truncate(true)
-            .open(path)?;
+        // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
+        #[allow(clippy::suspicious_open_options)]
+        let file = OpenOptions::new().write(true).read(true).create(true).open(path)?;
 
         let inner = Arc::new(TombstoneLogInner {
             inflights: Mutex::new(vec![]),

From 48c39900f98da9c51e3cba81e7d79d9ab2eb6716 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 11 Apr 2024 18:06:16 +0800
Subject: [PATCH 251/261] chore: bump foyer to 0.7.0 (#323)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md                        | 24 ++++++++++++++++++++++++
 foyer-common/Cargo.toml             |  4 ++--
 foyer-experimental-bench/Cargo.toml | 13 +++++++------
 foyer-experimental/Cargo.toml       |  7 ++++---
 foyer-intrusive/Cargo.toml          |  6 +++---
 foyer-memory/Cargo.toml             |  6 +++---
 foyer-storage-bench/Cargo.toml      | 10 +++++-----
 foyer-storage/Cargo.toml            |  8 ++++----
 foyer-workspace-hack/Cargo.toml     |  2 +-
 foyer/Cargo.toml                    | 12 ++++++------
 10 files changed, 59 insertions(+), 33 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2abd56b2..3e656e0b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,23 @@
+## 2024-04-11
+
+| crate | version |
+| - | - |
+| foyer | 0.7.0 |
+| foyer-common | 0.5.0 |
+| foyer-intrusive | 0.4.0 |
+| foyer-memory | 0.2.0 |
+| foyer-storage | 0.6.0 |
+| foyer-storage-bench | 0.6.0 |
+| foyer-workspace-hack | 0.4.0 |
+
+<details>
+
+### Changes
+
+- Make `foyer` compatible with rust stable toolchain (MSRV = 1.77.2). 🎉
+
+</details>
+
 ## 2024-04-09
 
 | crate | version |
@@ -12,6 +32,8 @@
 - fix: Fix panics on `state()` for s3fifo entry.
 - fix: Enable `offset_of` feature for `foyer-storage`.
 
+</details>
+
 ## 2024-04-08
 
 | crate | version |
@@ -26,6 +48,8 @@
 - feat: Introduce s3fifo to `foyer-memory`.
 - fix: Fix doctest for `foyer-intrusive`.
 
+</details>
+
 ## 2024-03-21
 
 | crate | version |
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 0abd488a..335ae28e 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-common"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "common utils for foyer - the hybrid cache for Rust"
@@ -17,7 +17,7 @@ normal = ["foyer-workspace-hack"]
 anyhow = "1.0"
 bytes = "1"
 cfg-if = "1"
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
diff --git a/foyer-experimental-bench/Cargo.toml b/foyer-experimental-bench/Cargo.toml
index 4552d34d..b21e44e2 100644
--- a/foyer-experimental-bench/Cargo.toml
+++ b/foyer-experimental-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-experimental-bench"
-version = "0.1.0"
+version = "0.0.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine bench tool for foyer - the hybrid cache for Rust"
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
 readme = "../README.md"
+publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 autobenches = false
 
@@ -19,11 +20,11 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
-foyer-common = { version = "0.4", path = "../foyer-common" }
-foyer-experimental = { version = "0.1", path = "../foyer-experimental" }
-foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.5", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.5", path = "../foyer-common" }
+foyer-experimental = { version = "*", path = "../foyer-experimental" }
+foyer-intrusive = { version = "0.4", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.6", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
 http-body-util = "0.1"
diff --git a/foyer-experimental/Cargo.toml b/foyer-experimental/Cargo.toml
index ab8b0a50..f60cac62 100644
--- a/foyer-experimental/Cargo.toml
+++ b/foyer-experimental/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-experimental"
-version = "0.1.0"
+version = "0.0.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "experimental components for foyer - the hybrid cache for Rust"
@@ -8,6 +8,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mrcroxx/foyer"
 homepage = "https://github.com/mrcroxx/foyer"
 readme = "../README.md"
+publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-udeps.ignore]
@@ -17,8 +18,8 @@ normal = ["foyer-workspace-hack"]
 anyhow = "1.0"
 bytes = "1"
 crossbeam = { version = "0.8", features = ["std", "crossbeam-channel"] }
-foyer-common = { version = "0.4", path = "../foyer-common" }
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.5", path = "../foyer-common" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }
 lazy_static = "1"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1.0"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 8b33072a..f178e839 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-intrusive"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "intrusive data structures for foyer - the hybrid cache for Rust"
@@ -16,8 +16,8 @@ normal = ["foyer-workspace-hack"]
 [dependencies]
 bytes = "1"
 cmsketch = "0.1"
-foyer-common = { version = "0.4", path = "../foyer-common" }
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.5", path = "../foyer-common" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }
 itertools = "0.12"
 memoffset = "0.9"
 parking_lot = "0.12"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 9c26a60c..a37f336c 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.1.4"
+version = "0.2.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"
@@ -18,8 +18,8 @@ ahash = "0.8"
 bitflags = "2"
 cmsketch = "0.2"
 crossbeam = "0.8"
-foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-intrusive = { version = "0.4", path = "../foyer-intrusive" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hashbrown = "0.14"
 itertools = "0.12"
diff --git a/foyer-storage-bench/Cargo.toml b/foyer-storage-bench/Cargo.toml
index e8400682..c7f02b49 100644
--- a/foyer-storage-bench/Cargo.toml
+++ b/foyer-storage-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage-bench"
-version = "0.5.1"
+version = "0.6.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine bench tool for foyer - the hybrid cache for Rust"
@@ -18,10 +18,10 @@ anyhow = "1"
 bytesize = "1"
 clap = { version = "4", features = ["derive"] }
 console-subscriber = { version = "0.2", optional = true }
-foyer-common = { version = "0.4", path = "../foyer-common" }
-foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
-foyer-storage = { version = "0.5", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.5", path = "../foyer-common" }
+foyer-intrusive = { version = "0.4", path = "../foyer-intrusive" }
+foyer-storage = { version = "0.6", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }
 futures = "0.3"
 hdrhistogram = "7"
 http-body-util = "0.1"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 4eb2174a..16bba14c 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.5.1"
+version = "0.6.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
@@ -19,9 +19,9 @@ anyhow = "1.0"
 bitflags = "2.3.1"
 bitmaps = "3.2"
 bytes = "1"
-foyer-common = { version = "0.4", path = "../foyer-common" }
-foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.5", path = "../foyer-common" }
+foyer-intrusive = { version = "0.4", path = "../foyer-intrusive" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }
 futures = "0.3"
 itertools = "0.12"
 lazy_static = "1"
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index 03c4afc0..b03784d9 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -4,7 +4,7 @@
 
 [package]
 name = "foyer-workspace-hack"
-version = "0.3.0"
+version = "0.4.0"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "workspace-hack package, managed by hakari"
 license = "Apache-2.0"
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 4c9fb883..5e3652c2 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -15,8 +15,8 @@ rust-version = "1.77.2"
 normal = ["foyer-workspace-hack"]
 
 [dependencies]
-foyer-common = { version = "0.4", path = "../foyer-common" }
-foyer-intrusive = { version = "0.3", path = "../foyer-intrusive" }
-foyer-memory = { version = "0.1", path = "../foyer-memory" }
-foyer-storage = { version = "0.5", path = "../foyer-storage" }
-foyer-workspace-hack = { version = "0.3", path = "../foyer-workspace-hack" }
+foyer-common = { version = "0.5", path = "../foyer-common" }
+foyer-intrusive = { version = "0.4", path = "../foyer-intrusive" }
+foyer-memory = { version = "0.2", path = "../foyer-memory" }
+foyer-storage = { version = "0.6", path = "../foyer-storage" }
+foyer-workspace-hack = { version = "0.4", path = "../foyer-workspace-hack" }

From cc64fc951d9b5013ae850c4489b5fccd1350d0e3 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 12 Apr 2024 01:33:55 +0800
Subject: [PATCH 252/261] refactor: loose query key bound for in-memory cache
 (#324)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 examples/memory.rs          |  2 +-
 foyer-memory/src/cache.rs   | 19 ++++++++++++++++---
 foyer-memory/src/generic.rs | 26 ++++++++++++++++++++++----
 foyer-memory/src/indexer.rs | 37 +++++++++++++++++++++++++++----------
 4 files changed, 66 insertions(+), 18 deletions(-)

diff --git a/examples/memory.rs b/examples/memory.rs
index db8be486..69a76cb9 100644
--- a/examples/memory.rs
+++ b/examples/memory.rs
@@ -29,7 +29,7 @@ fn main() {
     });
 
     let entry = cache.insert("hello".to_string(), "world".to_string(), 1);
-    let e = cache.get(&"hello".to_string()).unwrap();
+    let e = cache.get("hello").unwrap();
 
     assert_eq!(entry.value(), e.value());
 }
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 269671e9..acb16b74 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -12,7 +12,12 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{hash::BuildHasher, ops::Deref, sync::Arc};
+use std::{
+    borrow::Borrow,
+    hash::{BuildHasher, Hash},
+    ops::Deref,
+    sync::Arc,
+};
 
 use ahash::RandomState;
 use futures::{Future, FutureExt};
@@ -293,7 +298,11 @@ where
         }
     }
 
-    pub fn remove(&self, key: &K) {
+    pub fn remove<Q>(&self, key: &Q)
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
         match self {
             Cache::Fifo(cache) => cache.remove(key),
             Cache::Lru(cache) => cache.remove(key),
@@ -302,7 +311,11 @@ where
         }
     }
 
-    pub fn get(&self, key: &K) -> Option<CacheEntry<K, V, L, S>> {
+    pub fn get<Q>(&self, key: &Q) -> Option<CacheEntry<K, V, L, S>>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
         match self {
             Cache::Fifo(cache) => cache.get(key).map(CacheEntry::from),
             Cache::Lru(cache) => cache.get(key).map(CacheEntry::from),
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index 5ac43267..f50f59ad 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -13,8 +13,10 @@
 //  limitations under the License.
 
 use std::{
+    borrow::Borrow,
     future::Future,
     hash::BuildHasher,
+    hash::Hash,
     ops::Deref,
     ptr::NonNull,
     sync::{
@@ -138,7 +140,11 @@ where
         ptr
     }
 
-    unsafe fn get(&mut self, hash: u64, key: &K) -> Option<NonNull<H>> {
+    unsafe fn get<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<H>>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
         let mut ptr = match self.indexer.get(hash, key) {
             Some(ptr) => {
                 self.state.metrics.hit.fetch_add(1, Ordering::Relaxed);
@@ -161,7 +167,11 @@ where
     /// Remove a key from the cache.
     ///
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn remove(&mut self, hash: u64, key: &K) -> Option<(K, V, H::Context, usize)> {
+    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<(K, V, H::Context, usize)>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
         let ptr = self.indexer.remove(hash, key)?;
         self.state.metrics.remove.fetch_add(1, Ordering::Relaxed);
         if ptr.as_ref().base().is_in_eviction() {
@@ -472,7 +482,11 @@ where
         entry
     }
 
-    pub fn remove(&self, key: &K) {
+    pub fn remove<Q>(&self, key: &Q)
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
         let hash = self.hash_builder.hash_one(key);
 
         let entry = unsafe {
@@ -486,7 +500,11 @@ where
         }
     }
 
-    pub fn get(self: &Arc<Self>, key: &K) -> Option<GenericCacheEntry<K, V, H, E, I, L, S>> {
+    pub fn get<Q>(self: &Arc<Self>, key: &Q) -> Option<GenericCacheEntry<K, V, H, E, I, L, S>>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
         let hash = self.hash_builder.hash_one(key);
 
         unsafe {
diff --git a/foyer-memory/src/indexer.rs b/foyer-memory/src/indexer.rs
index b22cde15..17e29a8d 100644
--- a/foyer-memory/src/indexer.rs
+++ b/foyer-memory/src/indexer.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::ptr::NonNull;
+use std::{borrow::Borrow, hash::Hash, ptr::NonNull};
 
 use hashbrown::hash_table::{Entry as HashTableEntry, HashTable};
 
@@ -24,8 +24,14 @@ pub trait Indexer: Send + Sync + 'static {
 
     fn new() -> Self;
     unsafe fn insert(&mut self, handle: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>>;
-    unsafe fn get(&self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>>;
-    unsafe fn remove(&mut self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>>;
+    unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+    where
+        Self::Key: Borrow<Q>,
+        Q: Hash + Eq + ?Sized;
+    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+    where
+        Self::Key: Borrow<Q>,
+        Q: Hash + Eq + ?Sized;
     unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>>;
 }
 
@@ -90,15 +96,26 @@ where
         }
     }
 
-    unsafe fn get(&self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>> {
-        self.table.find(hash, |p| p.as_ref().base().key() == key).copied()
+    unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+    where
+        Self::Key: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        self.table
+            .find(hash, |p| p.as_ref().base().key().borrow() == key)
+            .copied()
     }
 
-    unsafe fn remove(&mut self, hash: u64, key: &Self::Key) -> Option<NonNull<Self::Handle>> {
-        match self
-            .table
-            .entry(hash, |p| p.as_ref().base().key() == key, |p| p.as_ref().base().hash())
-        {
+    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+    where
+        Self::Key: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        match self.table.entry(
+            hash,
+            |p| p.as_ref().base().key().borrow() == key,
+            |p| p.as_ref().base().hash(),
+        ) {
             HashTableEntry::Occupied(o) => {
                 let (mut p, _) = o.remove();
                 let b = p.as_mut().base_mut();

From e47c0909cf91a4421012c154d285914adebe64f5 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 12 Apr 2024 01:49:21 +0800
Subject: [PATCH 253/261] feat: impl contains for in-memory cache (#325)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs   | 13 +++++++++++++
 foyer-memory/src/generic.rs | 21 +++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index acb16b74..5739fd12 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -324,6 +324,19 @@ where
         }
     }
 
+    pub fn contains<Q>(&self, key: &Q) -> bool
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        match self {
+            Cache::Fifo(cache) => cache.contains(key),
+            Cache::Lru(cache) => cache.contains(key),
+            Cache::Lfu(cache) => cache.contains(key),
+            Cache::S3Fifo(cache) => cache.contains(key),
+        }
+    }
+
     pub fn clear(&self) {
         match self {
             Cache::Fifo(cache) => cache.clear(),
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index f50f59ad..c39c2585 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -164,6 +164,14 @@ where
         Some(ptr)
     }
 
+    unsafe fn contains<Q>(&mut self, hash: u64, key: &Q) -> bool
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        self.indexer.get(hash, key).is_some()
+    }
+
     /// Remove a key from the cache.
     ///
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
@@ -516,6 +524,19 @@ where
         }
     }
 
+    pub fn contains<Q>(self: &Arc<Self>, key: &Q) -> bool
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        let hash = self.hash_builder.hash_one(key);
+
+        unsafe {
+            let mut shard = self.shards[hash as usize % self.shards.len()].lock();
+            shard.contains(hash, key)
+        }
+    }
+
     pub fn clear(&self) {
         let mut to_deallocate = vec![];
         for shard in self.shards.iter() {

From 818e3f92b7a2729999519320c31e09938958411f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 12 Apr 2024 14:41:34 +0800
Subject: [PATCH 254/261] perf: add dynamic dispatch bench (#326)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/Cargo.toml                       |   4 +
 .../benches/bench_dynamic_dispatch.rs         | 121 ++++++++++++++++++
 2 files changed, 125 insertions(+)
 create mode 100644 foyer-memory/benches/bench_dynamic_dispatch.rs

diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index a37f336c..aaf48453 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -43,3 +43,7 @@ deadlock = ["parking_lot/deadlock_detection"]
 [[bench]]
 name = "bench_hit_ratio"
 harness = false
+
+[[bench]]
+name = "bench_dynamic_dispatch"
+harness = false
\ No newline at end of file
diff --git a/foyer-memory/benches/bench_dynamic_dispatch.rs b/foyer-memory/benches/bench_dynamic_dispatch.rs
new file mode 100644
index 00000000..1e92d5c9
--- /dev/null
+++ b/foyer-memory/benches/bench_dynamic_dispatch.rs
@@ -0,0 +1,121 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+
+struct T<F>
+where
+    F: Fn(&str) -> usize,
+{
+    f1: F,
+    f2: Box<dyn Fn(&str) -> usize>,
+    f3: Arc<dyn Fn(&str) -> usize>,
+}
+
+fn rand_string(len: usize) -> String {
+    thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(len)
+        .map(char::from)
+        .collect()
+}
+
+fn bench_static_dispatch<F>(t: &T<F>, loops: usize) -> Duration
+where
+    F: Fn(&str) -> usize,
+{
+    let mut dur = Duration::default();
+    for _ in 0..loops {
+        let s = rand_string(thread_rng().gen_range(0..100));
+        let now = Instant::now();
+        let _ = (t.f1)(&s);
+        dur += now.elapsed();
+    }
+    Duration::from_nanos((dur.as_nanos() as usize / loops) as _)
+}
+
+fn bench_box_dynamic_dispatch<F>(t: &T<F>, loops: usize) -> Duration
+where
+    F: Fn(&str) -> usize,
+{
+    let mut dur = Duration::default();
+    for _ in 0..loops {
+        let s = rand_string(thread_rng().gen_range(0..100));
+        let now = Instant::now();
+        let _ = (t.f3)(&s);
+        dur += now.elapsed();
+    }
+    Duration::from_nanos((dur.as_nanos() as usize / loops) as _)
+}
+
+fn bench_arc_dynamic_dispatch<F>(t: &T<F>, loops: usize) -> Duration
+where
+    F: Fn(&str) -> usize,
+{
+    let mut dur = Duration::default();
+    for _ in 0..loops {
+        let s = rand_string(thread_rng().gen_range(0..100));
+        let now = Instant::now();
+        let _ = (t.f2)(&s);
+        dur += now.elapsed();
+    }
+    Duration::from_nanos((dur.as_nanos() as usize / loops) as _)
+}
+
+fn main() {
+    let t = T {
+        f1: |s: &str| s.len(),
+        f2: Box::new(|s: &str| s.len()),
+        f3: Arc::new(|s: &str| s.len()),
+    };
+
+    let _ = T {
+        f1: |s: &str| s.len(),
+        f2: Box::new(|s: &str| s.len() + 1),
+        f3: Arc::new(|s: &str| s.len() + 1),
+    };
+
+    let _ = T {
+        f1: |s: &str| s.len(),
+        f2: Box::new(|s: &str| s.len() + 2),
+        f3: Arc::new(|s: &str| s.len() + 2),
+    };
+
+    let _ = T {
+        f1: |s: &str| s.len(),
+        f2: Box::new(|s: &str| s.len() + 3),
+        f3: Arc::new(|s: &str| s.len() + 3),
+    };
+
+    for loops in [100_000, 1_000_000, 10_000_000] {
+        println!();
+
+        println!("     statis - {} loops : {:?}", loops, bench_static_dispatch(&t, loops));
+        println!(
+            "box dynamic - {} loops : {:?}",
+            loops,
+            bench_box_dynamic_dispatch(&t, loops)
+        );
+        println!(
+            "arc dynamic - {} loops : {:?}",
+            loops,
+            bench_arc_dynamic_dispatch(&t, loops)
+        );
+    }
+}

From 3fda8cc663f9479870bd6596936b02bbc69ad0de Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 12 Apr 2024 20:30:16 +0800
Subject: [PATCH 255/261] chore: fix ci coverage (#331)

* chore: fix ci coverage

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix check

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      | 1 +
 .github/workflows/main.yml         | 1 +
 .github/workflows/pull-request.yml | 1 +
 foyer-memory/Cargo.toml            | 2 +-
 4 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 5b368c44..8164b0c7 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -95,6 +95,7 @@ jobs:
         run: |
           cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v3
+        if: matrix.os == 'ubuntu-latest'
         with:
           verbose: true
   deadlock:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8107c45d..d9efce08 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -102,6 +102,7 @@ jobs:
         run: |
           cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v3
+        if: matrix.os == 'ubuntu-latest'
         with:
           verbose: true
   deadlock:
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 5d813c93..3d73dacf 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -101,6 +101,7 @@ jobs:
         run: |
           cargo llvm-cov report --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v3
+        if: matrix.os == 'ubuntu-latest'
         with:
           verbose: true
   deadlock:
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index aaf48453..2eb1ef52 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -46,4 +46,4 @@ harness = false
 
 [[bench]]
 name = "bench_dynamic_dispatch"
-harness = false
\ No newline at end of file
+harness = false

From 6b820faa40eb7b39255ae3ebddc5def8e00bafb9 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 12 Apr 2024 20:48:17 +0800
Subject: [PATCH 256/261] refactor: refine generic handle type (#330)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs   |  24 ++---
 foyer-memory/src/generic.rs | 201 +++++++++++++++++++-----------------
 2 files changed, 120 insertions(+), 105 deletions(-)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 5739fd12..d570535c 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -39,37 +39,37 @@ use crate::{
 };
 
 pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+    GenericCache<K, V, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
 pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Fifo<K, V>, L, S>;
 pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
 pub type FifoEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, FifoHandle<K, V>, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S, ER>;
 
 pub type LruCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+    GenericCache<K, V, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
 pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lru<K, V>, L, S>;
 pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
 pub type LruEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, LruHandle<K, V>, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S, ER>;
 
 pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+    GenericCache<K, V, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
 pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lfu<K, V>, L, S>;
 pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
 pub type LfuEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, LfuHandle<K, V>, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S, ER>;
 
 pub type S3FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, S3FifoHandle<K, V>, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
+    GenericCache<K, V, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
 pub type S3FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
     CacheConfig<S3Fifo<K, V>, L, S>;
 pub type S3FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, S3FifoHandle<K, V>, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
 pub type S3FifoEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, S3FifoHandle<K, V>, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S, ER>;
 
 pub enum CacheEntry<K, V, L, S = RandomState>
 where
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index c39c2585..d217fe12 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -47,13 +47,13 @@ struct CacheSharedState<T, L> {
 
 // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
 #[allow(clippy::type_complexity)]
-struct CacheShard<K, V, H, E, I, L, S>
+struct CacheShard<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -63,18 +63,18 @@ where
     capacity: usize,
     usage: Arc<AtomicUsize>,
 
-    waiters: HashMap<K, Vec<oneshot::Sender<GenericCacheEntry<K, V, H, E, I, L, S>>>>,
+    waiters: HashMap<K, Vec<oneshot::Sender<GenericCacheEntry<K, V, E, I, L, S>>>>,
 
-    state: Arc<CacheSharedState<H, L>>,
+    state: Arc<CacheSharedState<E::Handle, L>>,
 }
 
-impl<K, V, H, E, I, L, S> CacheShard<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> CacheShard<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -82,7 +82,7 @@ where
         capacity: usize,
         eviction_config: &E::Config,
         usage: Arc<AtomicUsize>,
-        context: Arc<CacheSharedState<H, L>>,
+        context: Arc<CacheSharedState<E::Handle, L>>,
     ) -> Self {
         let indexer = I::new();
         let eviction = unsafe { E::new(capacity, eviction_config) };
@@ -104,10 +104,14 @@ where
         key: K,
         value: V,
         charge: usize,
-        context: H::Context,
-        last_reference_entries: &mut Vec<(K, V, H::Context, usize)>,
-    ) -> NonNull<H> {
-        let mut handle = self.state.object_pool.pop().unwrap_or_else(|| Box::new(H::new()));
+        context: <E::Handle as Handle>::Context,
+        last_reference_entries: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>,
+    ) -> NonNull<E::Handle> {
+        let mut handle = self
+            .state
+            .object_pool
+            .pop()
+            .unwrap_or_else(|| Box::new(E::Handle::new()));
         handle.init(hash, key, value, charge, context);
         let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
 
@@ -140,7 +144,7 @@ where
         ptr
     }
 
-    unsafe fn get<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<H>>
+    unsafe fn get<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<E::Handle>>
     where
         K: Borrow<Q>,
         Q: Hash + Eq + ?Sized,
@@ -175,7 +179,7 @@ where
     /// Remove a key from the cache.
     ///
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<(K, V, H::Context, usize)>
+    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<(K, V, <E::Handle as Handle>::Context, usize)>
     where
         K: Borrow<Q>,
         Q: Hash + Eq + ?Sized,
@@ -191,7 +195,7 @@ where
     }
 
     /// Clear all cache entries.
-    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
+    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>) {
         // TODO(MrCroxx): Avoid collecting here?
         let ptrs = self.indexer.drain().collect_vec();
         let eptrs = self.eviction.clear();
@@ -216,7 +220,11 @@ where
         }
     }
 
-    unsafe fn evict(&mut self, charge: usize, last_reference_entries: &mut Vec<(K, V, H::Context, usize)>) {
+    unsafe fn evict(
+        &mut self,
+        charge: usize,
+        last_reference_entries: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>,
+    ) {
         // TODO(MrCroxx): Use `let_chains` here after it is stable.
         while self.usage.load(Ordering::Relaxed) + charge > self.capacity {
             let evicted = match self.eviction.pop() {
@@ -236,7 +244,10 @@ where
     /// Release a handle used by an external user.
     ///
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn try_release_external_handle(&mut self, mut ptr: NonNull<H>) -> Option<(K, V, H::Context, usize)> {
+    unsafe fn try_release_external_handle(
+        &mut self,
+        mut ptr: NonNull<E::Handle>,
+    ) -> Option<(K, V, <E::Handle as Handle>::Context, usize)> {
         ptr.as_mut().base_mut().dec_refs();
         self.try_release_handle(ptr, true)
     }
@@ -246,7 +257,11 @@ where
     /// Return the entry if the handle is released.
     ///
     /// Recycle it if possible.
-    unsafe fn try_release_handle(&mut self, mut ptr: NonNull<H>, reinsert: bool) -> Option<(K, V, H::Context, usize)> {
+    unsafe fn try_release_handle(
+        &mut self,
+        mut ptr: NonNull<E::Handle>,
+        reinsert: bool,
+    ) -> Option<(K, V, <E::Handle as Handle>::Context, usize)> {
         let base = ptr.as_mut().base_mut();
 
         if base.has_refs() {
@@ -297,13 +312,13 @@ where
     }
 }
 
-impl<K, V, H, E, I, L, S> Drop for CacheShard<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> Drop for CacheShard<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -328,30 +343,30 @@ where
 
 // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
 #[allow(clippy::type_complexity)]
-pub enum GenericEntry<K, V, H, E, I, L, S, ER>
+pub enum GenericEntry<K, V, E, I, L, S, ER>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
 {
     Invalid,
-    Hit(GenericCacheEntry<K, V, H, E, I, L, S>),
-    Wait(oneshot::Receiver<GenericCacheEntry<K, V, H, E, I, L, S>>),
-    Miss(JoinHandle<std::result::Result<GenericCacheEntry<K, V, H, E, I, L, S>, ER>>),
+    Hit(GenericCacheEntry<K, V, E, I, L, S>),
+    Wait(oneshot::Receiver<GenericCacheEntry<K, V, E, I, L, S>>),
+    Miss(JoinHandle<std::result::Result<GenericCacheEntry<K, V, E, I, L, S>, ER>>),
 }
 
-impl<K, V, H, E, I, L, S, ER> Default for GenericEntry<K, V, H, E, I, L, S, ER>
+impl<K, V, E, I, L, S, ER> Default for GenericEntry<K, V, E, I, L, S, ER>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
@@ -361,18 +376,18 @@ where
     }
 }
 
-impl<K, V, H, E, I, L, S, ER> Future for GenericEntry<K, V, H, E, I, L, S, ER>
+impl<K, V, E, I, L, S, ER> Future for GenericEntry<K, V, E, I, L, S, ER>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error + From<oneshot::error::RecvError>,
 {
-    type Output = std::result::Result<GenericCacheEntry<K, V, H, E, I, L, S>, ER>;
+    type Output = std::result::Result<GenericCacheEntry<K, V, E, I, L, S>, ER>;
 
     fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
         match &mut *self {
@@ -389,33 +404,33 @@ where
 
 // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
 #[allow(clippy::type_complexity)]
-pub struct GenericCache<K, V, H, E, I, L, S = RandomState>
+pub struct GenericCache<K, V, E, I, L, S = RandomState>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    shards: Vec<Mutex<CacheShard<K, V, H, E, I, L, S>>>,
+    shards: Vec<Mutex<CacheShard<K, V, E, I, L, S>>>,
 
     capacity: usize,
     usages: Vec<Arc<AtomicUsize>>,
 
-    context: Arc<CacheSharedState<H, L>>,
+    context: Arc<CacheSharedState<E::Handle, L>>,
 
     hash_builder: S,
 }
 
-impl<K, V, H, E, I, L, S> GenericCache<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> GenericCache<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -444,7 +459,7 @@ where
         }
     }
 
-    pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> GenericCacheEntry<K, V, H, E, I, L, S> {
+    pub fn insert(self: &Arc<Self>, key: K, value: V, charge: usize) -> GenericCacheEntry<K, V, E, I, L, S> {
         self.insert_with_context(key, value, charge, CacheContext::default())
     }
 
@@ -454,7 +469,7 @@ where
         value: V,
         charge: usize,
         context: CacheContext,
-    ) -> GenericCacheEntry<K, V, H, E, I, L, S> {
+    ) -> GenericCacheEntry<K, V, E, I, L, S> {
         let hash = self.hash_builder.hash_one(&key);
 
         let mut to_deallocate = vec![];
@@ -508,7 +523,7 @@ where
         }
     }
 
-    pub fn get<Q>(self: &Arc<Self>, key: &Q) -> Option<GenericCacheEntry<K, V, H, E, I, L, S>>
+    pub fn get<Q>(self: &Arc<Self>, key: &Q) -> Option<GenericCacheEntry<K, V, E, I, L, S>>
     where
         K: Borrow<Q>,
         Q: Hash + Eq + ?Sized,
@@ -557,7 +572,7 @@ where
         &self.context.metrics
     }
 
-    unsafe fn try_release_external_handle(&self, ptr: NonNull<H>) {
+    unsafe fn try_release_external_handle(&self, ptr: NonNull<E::Handle>) {
         let entry = {
             let base = ptr.as_ref().base();
             let mut shard = self.shards[base.hash() as usize % self.shards.len()].lock();
@@ -572,17 +587,17 @@ where
 }
 
 // TODO(MrCroxx): use `hashbrown::HashTable` with `Handle` may relax the `Clone` bound?
-impl<K, V, H, E, I, L, S> GenericCache<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> GenericCache<K, V, E, I, L, S>
 where
     K: Key + Clone,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> GenericEntry<K, V, H, E, I, L, S, ER>
+    pub fn entry<F, FU, ER>(self: &Arc<Self>, key: K, f: F) -> GenericEntry<K, V, E, I, L, S, ER>
     where
         F: FnOnce() -> FU,
         FU: Future<Output = std::result::Result<(V, usize, CacheContext), ER>> + Send + 'static,
@@ -633,39 +648,39 @@ where
     }
 }
 
-pub struct GenericCacheEntry<K, V, H, E, I, L, S = RandomState>
+pub struct GenericCacheEntry<K, V, E, I, L, S = RandomState>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    cache: Arc<GenericCache<K, V, H, E, I, L, S>>,
-    ptr: NonNull<H>,
+    cache: Arc<GenericCache<K, V, E, I, L, S>>,
+    ptr: NonNull<E::Handle>,
 }
 
-impl<K, V, H, E, I, L, S> GenericCacheEntry<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> GenericCacheEntry<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn key(&self) -> &H::Key {
+    pub fn key(&self) -> &<E::Handle as Handle>::Key {
         unsafe { self.ptr.as_ref().base().key() }
     }
 
-    pub fn value(&self) -> &H::Value {
+    pub fn value(&self) -> &<E::Handle as Handle>::Value {
         unsafe { self.ptr.as_ref().base().value() }
     }
 
-    pub fn context(&self) -> &H::Context {
+    pub fn context(&self) -> &<E::Handle as Handle>::Context {
         unsafe { self.ptr.as_ref().base().context() }
     }
 
@@ -678,13 +693,13 @@ where
     }
 }
 
-impl<K, V, H, E, I, L, S> Clone for GenericCacheEntry<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> Clone for GenericCacheEntry<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -704,13 +719,13 @@ where
     }
 }
 
-impl<K, V, H, E, I, L, S> Drop for GenericCacheEntry<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> Drop for GenericCacheEntry<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -719,13 +734,13 @@ where
     }
 }
 
-impl<K, V, H, E, I, L, S> Deref for GenericCacheEntry<K, V, H, E, I, L, S>
+impl<K, V, E, I, L, S> Deref for GenericCacheEntry<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -736,24 +751,24 @@ where
     }
 }
 
-unsafe impl<K, V, H, E, I, L, S> Send for GenericCacheEntry<K, V, H, E, I, L, S>
+unsafe impl<K, V, E, I, L, S> Send for GenericCacheEntry<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
 }
-unsafe impl<K, V, H, E, I, L, S> Sync for GenericCacheEntry<K, V, H, E, I, L, S>
+unsafe impl<K, V, E, I, L, S> Sync for GenericCacheEntry<K, V, E, I, L, S>
 where
     K: Key,
     V: Value,
-    H: Handle<Key = K, Value = V>,
-    E: Eviction<Handle = H>,
-    I: Indexer<Key = K, Handle = H>,
+    E: Eviction,
+    E::Handle: Handle<Key = K, Value = V>,
+    I: Indexer<Key = K, Handle = E::Handle>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {

From f590f090c7938401d3898177caa01d412d2bfc04 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 14 Apr 2024 15:54:29 +0800
Subject: [PATCH 257/261] refactor: loose bound of in-memory eviction related
 traits (#334)

* refactor: remove Handle bound from Eviction

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: loose bound of in-memory eviction related traits

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs               |  35 +++---
 foyer-memory/src/eviction/fifo.rs       |  91 ++++++--------
 foyer-memory/src/eviction/lfu.rs        | 123 +++++++-----------
 foyer-memory/src/eviction/lru.rs        | 111 ++++++-----------
 foyer-memory/src/eviction/mod.rs        |  16 ++-
 foyer-memory/src/eviction/s3fifo.rs     | 114 +++++++----------
 foyer-memory/src/eviction/test_utils.rs |   7 +-
 foyer-memory/src/generic.rs             | 159 ++++++++++++------------
 foyer-memory/src/handle.rs              |  71 +++++------
 foyer-memory/src/indexer.rs             |  35 +++---
 10 files changed, 327 insertions(+), 435 deletions(-)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index d570535c..277f858f 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -39,37 +39,40 @@ use crate::{
 };
 
 pub type FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
-pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Fifo<K, V>, L, S>;
+    GenericCache<K, V, Fifo<(K, V)>, HashTableIndexer<K, FifoHandle<(K, V)>>, L, S>;
+pub type FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    CacheConfig<K, V, Fifo<(K, V)>, L, S>;
 pub type FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, Fifo<(K, V)>, HashTableIndexer<K, FifoHandle<(K, V)>>, L, S>;
 pub type FifoEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, Fifo<K, V>, HashTableIndexer<K, FifoHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, Fifo<(K, V)>, HashTableIndexer<K, FifoHandle<(K, V)>>, L, S, ER>;
 
 pub type LruCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
-pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lru<K, V>, L, S>;
+    GenericCache<K, V, Lru<(K, V)>, HashTableIndexer<K, LruHandle<(K, V)>>, L, S>;
+pub type LruCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    CacheConfig<K, V, Lru<(K, V)>, L, S>;
 pub type LruCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, Lru<(K, V)>, HashTableIndexer<K, LruHandle<(K, V)>>, L, S>;
 pub type LruEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, Lru<K, V>, HashTableIndexer<K, LruHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, Lru<(K, V)>, HashTableIndexer<K, LruHandle<(K, V)>>, L, S, ER>;
 
 pub type LfuCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
-pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> = CacheConfig<Lfu<K, V>, L, S>;
+    GenericCache<K, V, Lfu<(K, V)>, HashTableIndexer<K, LfuHandle<(K, V)>>, L, S>;
+pub type LfuCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
+    CacheConfig<K, V, Lfu<(K, V)>, L, S>;
 pub type LfuCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, Lfu<(K, V)>, HashTableIndexer<K, LfuHandle<(K, V)>>, L, S>;
 pub type LfuEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, Lfu<K, V>, HashTableIndexer<K, LfuHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, Lfu<(K, V)>, HashTableIndexer<K, LfuHandle<(K, V)>>, L, S, ER>;
 
 pub type S3FifoCache<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCache<K, V, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
+    GenericCache<K, V, S3Fifo<(K, V)>, HashTableIndexer<K, S3FifoHandle<(K, V)>>, L, S>;
 pub type S3FifoCacheConfig<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    CacheConfig<S3Fifo<K, V>, L, S>;
+    CacheConfig<K, V, S3Fifo<(K, V)>, L, S>;
 pub type S3FifoCacheEntry<K, V, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericCacheEntry<K, V, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S>;
+    GenericCacheEntry<K, V, S3Fifo<(K, V)>, HashTableIndexer<K, S3FifoHandle<(K, V)>>, L, S>;
 pub type S3FifoEntry<K, V, ER, L = DefaultCacheEventListener<K, V>, S = RandomState> =
-    GenericEntry<K, V, S3Fifo<K, V>, HashTableIndexer<K, S3FifoHandle<K, V>>, L, S, ER>;
+    GenericEntry<K, V, S3Fifo<(K, V)>, HashTableIndexer<K, S3FifoHandle<(K, V)>>, L, S, ER>;
 
 pub enum CacheEntry<K, V, L, S = RandomState>
 where
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index c42e27cb..058cc715 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -22,7 +22,7 @@ use foyer_intrusive::{
 use crate::{
     eviction::Eviction,
     handle::{BaseHandle, Handle},
-    CacheContext, Key, Value,
+    CacheContext,
 };
 
 #[derive(Debug, Clone)]
@@ -40,34 +40,30 @@ impl From<FifoContext> for CacheContext {
     }
 }
 
-pub struct FifoHandle<K, V>
+pub struct FifoHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     link: DlistLink,
-    base: BaseHandle<K, V, FifoContext>,
+    base: BaseHandle<T, FifoContext>,
 }
 
-impl<K, V> Debug for FifoHandle<K, V>
+impl<T> Debug for FifoHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("FifoHandle").finish()
     }
 }
 
-intrusive_adapter! { FifoHandleDlistAdapter<K, V> = NonNull<FifoHandle<K, V>>: FifoHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+intrusive_adapter! { FifoHandleDlistAdapter<T> = NonNull<FifoHandle<T>>: FifoHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
 
-impl<K, V> Handle for FifoHandle<K, V>
+impl<T> Handle for FifoHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Key = K;
-    type Value = V;
+    type Data = T;
     type Context = FifoContext;
 
     fn new() -> Self {
@@ -77,15 +73,15 @@ where
         }
     }
 
-    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
-        self.base.init(hash, key, value, charge, context);
+    fn init(&mut self, hash: u64, data: Self::Data, charge: usize, context: Self::Context) {
+        self.base.init(hash, data, charge, context);
     }
 
-    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
         &self.base
     }
 
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
         &mut self.base
     }
 }
@@ -93,20 +89,18 @@ where
 #[derive(Debug, Clone)]
 pub struct FifoConfig {}
 
-pub struct Fifo<K, V>
+pub struct Fifo<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    queue: Dlist<FifoHandleDlistAdapter<K, V>>,
+    queue: Dlist<FifoHandleDlistAdapter<T>>,
 }
 
-impl<K, V> Eviction for Fifo<K, V>
+impl<T> Eviction for Fifo<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Handle = FifoHandle<K, V>;
+    type Item = FifoHandle<T>;
     type Config = FifoConfig;
 
     unsafe fn new(_capacity: usize, _config: &Self::Config) -> Self
@@ -116,29 +110,29 @@ where
         Self { queue: Dlist::new() }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Item>) {
         self.queue.push_back(ptr);
         ptr.as_mut().base_mut().set_in_eviction(true);
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Item>> {
         self.queue.pop_front().map(|mut ptr| {
             ptr.as_mut().base_mut().set_in_eviction(false);
             ptr
         })
     }
 
-    unsafe fn reinsert(&mut self, _: NonNull<Self::Handle>) {}
+    unsafe fn reinsert(&mut self, _: NonNull<Self::Item>) {}
 
-    unsafe fn access(&mut self, _: NonNull<Self::Handle>) {}
+    unsafe fn access(&mut self, _: NonNull<Self::Item>) {}
 
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Item>) {
         let p = self.queue.iter_mut_from_raw(ptr.as_mut().link.raw()).remove().unwrap();
         assert_eq!(p, ptr);
         ptr.as_mut().base_mut().set_in_eviction(false);
     }
 
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Item>> {
         let mut res = Vec::with_capacity(self.len());
         while let Some(mut ptr) = self.queue.pop_front() {
             ptr.as_mut().base_mut().set_in_eviction(false);
@@ -156,18 +150,8 @@ where
     }
 }
 
-unsafe impl<K, V> Send for Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-unsafe impl<K, V> Sync for Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
+unsafe impl<T> Send for Fifo<T> where T: Send + Sync + 'static {}
+unsafe impl<T> Sync for Fifo<T> where T: Send + Sync + 'static {}
 
 #[cfg(test)]
 pub mod tests {
@@ -177,25 +161,24 @@ pub mod tests {
     use super::*;
     use crate::eviction::test_utils::TestEviction;
 
-    impl<K, V> TestEviction for Fifo<K, V>
+    impl<T> TestEviction for Fifo<T>
     where
-        K: Key + Clone,
-        V: Value + Clone,
+        T: Send + Sync + 'static + Clone,
     {
-        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+        fn dump(&self) -> Vec<T> {
             self.queue
                 .iter()
-                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .map(|handle| handle.base().data_unwrap_unchecked().clone())
                 .collect_vec()
         }
     }
 
-    type TestFifoHandle = FifoHandle<u64, u64>;
-    type TestFifo = Fifo<u64, u64>;
+    type TestFifoHandle = FifoHandle<u64>;
+    type TestFifo = Fifo<u64>;
 
-    unsafe fn new_test_fifo_handle_ptr(key: u64, value: u64) -> NonNull<TestFifoHandle> {
+    unsafe fn new_test_fifo_handle_ptr(data: u64) -> NonNull<TestFifoHandle> {
         let mut handle = Box::new(TestFifoHandle::new());
-        handle.init(0, key, value, 1, FifoContext);
+        handle.init(0, data, 1, FifoContext);
         NonNull::new_unchecked(Box::into_raw(handle))
     }
 
@@ -206,7 +189,7 @@ pub mod tests {
     #[test]
     fn test_fifo() {
         unsafe {
-            let ptrs = (0..8).map(|i| new_test_fifo_handle_ptr(i, i)).collect_vec();
+            let ptrs = (0..8).map(|i| new_test_fifo_handle_ptr(i)).collect_vec();
 
             let mut fifo = TestFifo::new(100, &FifoConfig {});
 
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index e885a007..f1837562 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -24,7 +24,7 @@ use foyer_intrusive::{
 use crate::{
     eviction::Eviction,
     handle::{BaseHandle, Handle},
-    CacheContext, Key, Value,
+    CacheContext,
 };
 
 #[derive(Debug, Clone)]
@@ -68,35 +68,31 @@ enum Queue {
     Protected,
 }
 
-pub struct LfuHandle<K, V>
+pub struct LfuHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     link: DlistLink,
-    base: BaseHandle<K, V, LfuContext>,
+    base: BaseHandle<T, LfuContext>,
     queue: Queue,
 }
 
-impl<K, V> Debug for LfuHandle<K, V>
+impl<T> Debug for LfuHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LfuHandle").finish()
     }
 }
 
-intrusive_adapter! { LfuHandleDlistAdapter<K, V> = NonNull<LfuHandle<K, V>>: LfuHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+intrusive_adapter! { LfuHandleDlistAdapter<T> = NonNull<LfuHandle<T>>: LfuHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
 
-impl<K, V> Handle for LfuHandle<K, V>
+impl<T> Handle for LfuHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Key = K;
-    type Value = V;
+    type Data = T;
     type Context = LfuContext;
 
     fn new() -> Self {
@@ -107,31 +103,21 @@ where
         }
     }
 
-    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
-        self.base.init(hash, key, value, charge, context)
+    fn init(&mut self, hash: u64, data: Self::Data, charge: usize, context: Self::Context) {
+        self.base.init(hash, data, charge, context)
     }
 
-    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
         &self.base
     }
 
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
         &mut self.base
     }
 }
 
-unsafe impl<K, V> Send for LfuHandle<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-unsafe impl<K, V> Sync for LfuHandle<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
+unsafe impl<T> Send for LfuHandle<T> where T: Send + Sync + 'static {}
+unsafe impl<T> Sync for LfuHandle<T> where T: Send + Sync + 'static {}
 
 /// This implementation is inspired by [Caffeine](https://github.com/ben-manes/caffeine) under Apache License 2.0
 ///
@@ -145,14 +131,13 @@ where
 ///
 /// When evicting, the entry with a lower frequency from `window` or `probtion` will be evicted first, then from
 /// `protected`.
-pub struct Lfu<K, V>
+pub struct Lfu<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    window: Dlist<LfuHandleDlistAdapter<K, V>>,
-    probation: Dlist<LfuHandleDlistAdapter<K, V>>,
-    protected: Dlist<LfuHandleDlistAdapter<K, V>>,
+    window: Dlist<LfuHandleDlistAdapter<T>>,
+    probation: Dlist<LfuHandleDlistAdapter<T>>,
+    protected: Dlist<LfuHandleDlistAdapter<T>>,
 
     window_charges: usize,
     probation_charges: usize,
@@ -167,12 +152,11 @@ where
     decay: usize,
 }
 
-impl<K, V> Lfu<K, V>
+impl<T> Lfu<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    fn increase_queue_charges(&mut self, handle: &LfuHandle<K, V>) {
+    fn increase_queue_charges(&mut self, handle: &LfuHandle<T>) {
         let charges = handle.base().charge();
         match handle.queue {
             Queue::None => unreachable!(),
@@ -182,7 +166,7 @@ where
         }
     }
 
-    fn decrease_queue_charges(&mut self, handle: &LfuHandle<K, V>) {
+    fn decrease_queue_charges(&mut self, handle: &LfuHandle<T>) {
         let charges = handle.base().charge();
         match handle.queue {
             Queue::None => unreachable!(),
@@ -202,12 +186,11 @@ where
     }
 }
 
-impl<K, V> Eviction for Lfu<K, V>
+impl<T> Eviction for Lfu<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Handle = LfuHandle<K, V>;
+    type Item = LfuHandle<T>;
     type Config = LfuConfig;
 
     unsafe fn new(capacity: usize, config: &Self::Config) -> Self
@@ -252,7 +235,7 @@ where
         }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
 
         debug_assert!(!handle.link.is_linked());
@@ -278,7 +261,7 @@ where
         }
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Item>> {
         // Compare the frequency of the front element of `window` and `probation` queue, and evict the lower one.
         // If both `window` and `probation` are empty, try evict from `protected`.
         let mut ptr = match (self.window.front(), self.probation.front()) {
@@ -312,7 +295,7 @@ where
         Some(ptr)
     }
 
-    unsafe fn reinsert(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn reinsert(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
 
         match handle.queue {
@@ -361,11 +344,11 @@ where
         }
     }
 
-    unsafe fn access(&mut self, ptr: NonNull<Self::Handle>) {
+    unsafe fn access(&mut self, ptr: NonNull<Self::Item>) {
         self.update_frequencies(ptr.as_ref().base().hash());
     }
 
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
 
         debug_assert!(handle.link.is_linked());
@@ -386,7 +369,7 @@ where
         handle.base_mut().set_in_eviction(false);
     }
 
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Item>> {
         let mut res = Vec::with_capacity(self.len());
 
         while !self.is_empty() {
@@ -409,18 +392,8 @@ where
     }
 }
 
-unsafe impl<K, V> Send for Lfu<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-unsafe impl<K, V> Sync for Lfu<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
+unsafe impl<T> Send for Lfu<T> where T: Send + Sync + 'static {}
+unsafe impl<T> Sync for Lfu<T> where T: Send + Sync + 'static {}
 
 #[cfg(test)]
 mod tests {
@@ -430,23 +403,22 @@ mod tests {
     use super::*;
     use crate::eviction::test_utils::TestEviction;
 
-    impl<K, V> TestEviction for Lfu<K, V>
+    impl<T> TestEviction for Lfu<T>
     where
-        K: Key + Clone,
-        V: Value + Clone,
+        T: Send + Sync + 'static + Clone,
     {
-        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+        fn dump(&self) -> Vec<T> {
             self.window
                 .iter()
                 .chain(self.probation.iter())
                 .chain(self.protected.iter())
-                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .map(|handle| handle.base().data_unwrap_unchecked().clone())
                 .collect_vec()
         }
     }
 
-    type TestLfu = Lfu<u64, u64>;
-    type TestLfuHandle = LfuHandle<u64, u64>;
+    type TestLfu = Lfu<u64>;
+    type TestLfuHandle = LfuHandle<u64>;
 
     unsafe fn assert_test_lfu(
         lfu: &TestLfu,
@@ -463,14 +435,7 @@ mod tests {
         assert_eq!(lfu.window_charges, window);
         assert_eq!(lfu.probation_charges, probation);
         assert_eq!(lfu.protected_charges, protected);
-        let es = lfu
-            .dump()
-            .into_iter()
-            .map(|(k, v)| {
-                assert_eq!(k, v);
-                k
-            })
-            .collect_vec();
+        let es = lfu.dump().into_iter().collect_vec();
         assert_eq!(es, entries);
     }
 
@@ -485,7 +450,7 @@ mod tests {
             let ptrs = (0..100)
                 .map(|i| {
                     let mut handle = Box::new(TestLfuHandle::new());
-                    handle.init(i, i, i, 1, LfuContext);
+                    handle.init(i, i, 1, LfuContext);
                     NonNull::new_unchecked(Box::into_raw(handle))
                 })
                 .collect_vec();
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index 3cfc44d7..0e18a5d1 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -23,7 +23,7 @@ use foyer_intrusive::{
 use crate::{
     eviction::Eviction,
     handle::{BaseHandle, Handle},
-    CacheContext, Key, Value,
+    CacheContext,
 };
 
 #[derive(Debug, Clone)]
@@ -63,35 +63,31 @@ impl From<LruContext> for CacheContext {
     }
 }
 
-pub struct LruHandle<K, V>
+pub struct LruHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     link: DlistLink,
-    base: BaseHandle<K, V, LruContext>,
+    base: BaseHandle<T, LruContext>,
     in_high_priority_pool: bool,
 }
 
-impl<K, V> Debug for LruHandle<K, V>
+impl<T> Debug for LruHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LruHandle").finish()
     }
 }
 
-intrusive_adapter! { LruHandleDlistAdapter<K, V> = NonNull<LruHandle<K, V>>: LruHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+intrusive_adapter! { LruHandleDlistAdapter<T> = NonNull<LruHandle<T>>: LruHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
 
-impl<K, V> Handle for LruHandle<K, V>
+impl<T> Handle for LruHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Key = K;
-    type Value = V;
+    type Data = T;
     type Context = LruContext;
 
     fn new() -> Self {
@@ -102,48 +98,36 @@ where
         }
     }
 
-    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
-        self.base.init(hash, key, value, charge, context)
+    fn init(&mut self, hash: u64, data: Self::Data, charge: usize, context: Self::Context) {
+        self.base.init(hash, data, charge, context)
     }
 
-    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
         &self.base
     }
 
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
         &mut self.base
     }
 }
 
-unsafe impl<K, V> Send for LruHandle<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-unsafe impl<K, V> Sync for LruHandle<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
+unsafe impl<T> Send for LruHandle<T> where T: Send + Sync + 'static {}
+unsafe impl<T> Sync for LruHandle<T> where T: Send + Sync + 'static {}
 
-pub struct Lru<K, V>
+pub struct Lru<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    high_priority_list: Dlist<LruHandleDlistAdapter<K, V>>,
-    list: Dlist<LruHandleDlistAdapter<K, V>>,
+    high_priority_list: Dlist<LruHandleDlistAdapter<T>>,
+    list: Dlist<LruHandleDlistAdapter<T>>,
 
     high_priority_charges: usize,
     high_priority_charges_capacity: usize,
 }
 
-impl<K, V> Lru<K, V>
+impl<T> Lru<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     unsafe fn may_overflow_high_priority_pool(&mut self) {
         while self.high_priority_charges > self.high_priority_charges_capacity {
@@ -158,12 +142,11 @@ where
     }
 }
 
-impl<K, V> Eviction for Lru<K, V>
+impl<T> Eviction for Lru<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Handle = LruHandle<K, V>;
+    type Item = LruHandle<T>;
     type Config = LruConfig;
 
     unsafe fn new(capacity: usize, config: &Self::Config) -> Self
@@ -186,7 +169,7 @@ where
         }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
 
         debug_assert!(!handle.link.is_linked());
@@ -208,7 +191,7 @@ where
         handle.base_mut().set_in_eviction(true);
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Item>> {
         let mut ptr = self.list.pop_front().or_else(|| self.high_priority_list.pop_front())?;
 
         let handle = ptr.as_mut();
@@ -223,9 +206,9 @@ where
         Some(ptr)
     }
 
-    unsafe fn access(&mut self, _: NonNull<Self::Handle>) {}
+    unsafe fn access(&mut self, _: NonNull<Self::Item>) {}
 
-    unsafe fn reinsert(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn reinsert(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
 
         if handle.base().is_in_eviction() {
@@ -238,7 +221,7 @@ where
         }
     }
 
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
         debug_assert!(handle.link.is_linked());
 
@@ -252,7 +235,7 @@ where
         handle.base_mut().set_in_eviction(false);
     }
 
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Item>> {
         let mut res = Vec::with_capacity(self.len());
 
         while !self.list.is_empty() {
@@ -282,18 +265,8 @@ where
     }
 }
 
-unsafe impl<K, V> Send for Lru<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-unsafe impl<K, V> Sync for Lru<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
+unsafe impl<T> Send for Lru<T> where T: Send + Sync + 'static {}
+unsafe impl<T> Sync for Lru<T> where T: Send + Sync + 'static {}
 
 #[cfg(test)]
 pub mod tests {
@@ -304,26 +277,25 @@ pub mod tests {
     use super::*;
     use crate::eviction::test_utils::TestEviction;
 
-    impl<K, V> TestEviction for Lru<K, V>
+    impl<T> TestEviction for Lru<T>
     where
-        K: Key + Clone,
-        V: Value + Clone,
+        T: Send + Sync + 'static + Clone,
     {
-        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+        fn dump(&self) -> Vec<T> {
             self.list
                 .iter()
                 .chain(self.high_priority_list.iter())
-                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .map(|handle| handle.base().data_unwrap_unchecked().clone())
                 .collect_vec()
         }
     }
 
-    type TestLruHandle = LruHandle<u64, u64>;
-    type TestLru = Lru<u64, u64>;
+    type TestLruHandle = LruHandle<u64>;
+    type TestLru = Lru<u64>;
 
-    unsafe fn new_test_lru_handle_ptr(key: u64, value: u64, context: LruContext) -> NonNull<TestLruHandle> {
+    unsafe fn new_test_lru_handle_ptr(data: u64, context: LruContext) -> NonNull<TestLruHandle> {
         let mut handle = Box::new(TestLruHandle::new());
-        handle.init(0, key, value, 1, context);
+        handle.init(0, data, 1, context);
         NonNull::new_unchecked(Box::into_raw(handle))
     }
 
@@ -350,7 +322,6 @@ pub mod tests {
             let ptrs = (0..20)
                 .map(|i| {
                     new_test_lru_handle_ptr(
-                        i,
                         i,
                         if i < 10 {
                             LruContext::HighPriority
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index d1ba0c9d..4b9d4b09 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -14,13 +14,11 @@
 
 use std::ptr::NonNull;
 
-use crate::handle::Handle;
-
 /// The lifetime of `handle: Self::H` is managed by [`Indexer`].
 ///
 /// Each `handle`'s lifetime in [`Indexer`] must outlive the raw pointer in [`Eviction`].
 pub trait Eviction: Send + Sync + 'static {
-    type Handle: Handle;
+    type Item;
     type Config;
 
     /// Create a new empty eviction container.
@@ -39,7 +37,7 @@ pub trait Eviction: Send + Sync + 'static {
     /// The `ptr` must be kept holding until `pop` or `remove`.
     ///
     /// The base handle associated to the `ptr` must be set in cache.
-    unsafe fn push(&mut self, ptr: NonNull<Self::Handle>);
+    unsafe fn push(&mut self, ptr: NonNull<Self::Item>);
 
     /// Pop a handle `ptr` from the eviction container.
     ///
@@ -49,7 +47,7 @@ pub trait Eviction: Send + Sync + 'static {
     /// Or it may become dangling and cause UB.
     ///
     /// The base handle associated to the `ptr` must be set NOT in cache.
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>>;
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Item>>;
 
     /// Try to reinsert a handle `ptr` into the eviction container after access.
     ///
@@ -60,7 +58,7 @@ pub trait Eviction: Send + Sync + 'static {
     /// The given `ptr` may be either IN or NOT IN the eviction container.
     /// If the `ptr` is reinserted, the base handle associated to it must be set in cache.
     /// If the `ptr` is NOT reinserted, the base handle associated to it must be set NOT in cache.
-    unsafe fn reinsert(&mut self, ptr: NonNull<Self::Handle>);
+    unsafe fn reinsert(&mut self, ptr: NonNull<Self::Item>);
 
     /// Notify the eviciton container that the `ptr` is accessed.
     /// The eviction container can update its statistics.
@@ -68,7 +66,7 @@ pub trait Eviction: Send + Sync + 'static {
     /// # Safety
     ///
     /// The given `ptr` can be EITHER in the eviction container OR not in the eviction container.
-    unsafe fn access(&mut self, ptr: NonNull<Self::Handle>);
+    unsafe fn access(&mut self, ptr: NonNull<Self::Item>);
 
     /// Remove the given `ptr` from the eviction container.
     ///
@@ -79,7 +77,7 @@ pub trait Eviction: Send + Sync + 'static {
     /// The `ptr` must be taken from the eviction container, otherwise it may become dangling and cause UB.
     ///
     /// The base handle associated to the `ptr` must be set NOT in cache.
-    unsafe fn remove(&mut self, ptr: NonNull<Self::Handle>);
+    unsafe fn remove(&mut self, ptr: NonNull<Self::Item>);
 
     /// Remove all `ptr`s from the eviction container and reset.
     ///
@@ -88,7 +86,7 @@ pub trait Eviction: Send + Sync + 'static {
     /// All `ptr` must be taken from the eviction container, otherwise it may become dangling and cause UB.
     ///
     /// All base handles associated to the `ptr`s must be set NOT in cache.
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>>;
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Item>>;
 
     /// Return the count of the `ptr`s that in the eviction container.
     ///
diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs
index e8911494..d21a50bc 100644
--- a/foyer-memory/src/eviction/s3fifo.rs
+++ b/foyer-memory/src/eviction/s3fifo.rs
@@ -22,7 +22,7 @@ use foyer_intrusive::{
 use crate::{
     eviction::Eviction,
     handle::{BaseHandle, Handle},
-    CacheContext, Key, Value,
+    CacheContext,
 };
 
 #[derive(Debug, Clone)]
@@ -46,33 +46,30 @@ enum Queue {
     Small,
 }
 
-pub struct S3FifoHandle<K, V>
+pub struct S3FifoHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     link: DlistLink,
-    base: BaseHandle<K, V, S3FifoContext>,
+    base: BaseHandle<T, S3FifoContext>,
     freq: u8,
     queue: Queue,
 }
 
-impl<K, V> Debug for S3FifoHandle<K, V>
+impl<T> Debug for S3FifoHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("S3FifoHandle").finish()
     }
 }
 
-intrusive_adapter! { S3FifoHandleDlistAdapter<K, V> = NonNull<S3FifoHandle<K, V>>: S3FifoHandle<K, V> { link: DlistLink } where K: Key, V: Value }
+intrusive_adapter! { S3FifoHandleDlistAdapter<T> = NonNull<S3FifoHandle<T>>: S3FifoHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
 
-impl<K, V> S3FifoHandle<K, V>
+impl<T> S3FifoHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
     #[inline(always)]
     pub fn inc(&mut self) {
@@ -90,13 +87,11 @@ where
     }
 }
 
-impl<K, V> Handle for S3FifoHandle<K, V>
+impl<T> Handle for S3FifoHandle<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Key = K;
-    type Value = V;
+    type Data = T;
     type Context = S3FifoContext;
 
     fn new() -> Self {
@@ -108,15 +103,15 @@ where
         }
     }
 
-    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context) {
-        self.base.init(hash, key, value, charge, context);
+    fn init(&mut self, hash: u64, data: Self::Data, charge: usize, context: Self::Context) {
+        self.base.init(hash, data, charge, context);
     }
 
-    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
         &self.base
     }
 
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context> {
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
         &mut self.base
     }
 }
@@ -126,13 +121,12 @@ pub struct S3FifoConfig {
     pub small_queue_capacity_ratio: f64,
 }
 
-pub struct S3Fifo<K, V>
+pub struct S3Fifo<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    small_queue: Dlist<S3FifoHandleDlistAdapter<K, V>>,
-    main_queue: Dlist<S3FifoHandleDlistAdapter<K, V>>,
+    small_queue: Dlist<S3FifoHandleDlistAdapter<T>>,
+    main_queue: Dlist<S3FifoHandleDlistAdapter<T>>,
 
     small_capacity: usize,
 
@@ -140,12 +134,11 @@ where
     main_charges: usize,
 }
 
-impl<K, V> S3Fifo<K, V>
+impl<T> S3Fifo<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    unsafe fn evict(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
+    unsafe fn evict(&mut self) -> Option<NonNull<S3FifoHandle<T>>> {
         // TODO(MrCroxx): Use `let_chains` here after it is stable.
         if self.small_charges > self.small_capacity {
             if let Some(ptr) = self.evict_small() {
@@ -155,7 +148,7 @@ where
         self.evict_main()
     }
 
-    unsafe fn evict_small(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
+    unsafe fn evict_small(&mut self) -> Option<NonNull<S3FifoHandle<T>>> {
         while let Some(mut ptr) = self.small_queue.pop_front() {
             let handle = ptr.as_mut();
             if handle.freq > 1 {
@@ -173,7 +166,7 @@ where
         None
     }
 
-    unsafe fn evict_main(&mut self) -> Option<NonNull<<S3Fifo<K, V> as Eviction>::Handle>> {
+    unsafe fn evict_main(&mut self) -> Option<NonNull<S3FifoHandle<T>>> {
         while let Some(mut ptr) = self.main_queue.pop_front() {
             let handle = ptr.as_mut();
             if handle.freq > 0 {
@@ -189,12 +182,11 @@ where
     }
 }
 
-impl<K, V> Eviction for S3Fifo<K, V>
+impl<T> Eviction for S3Fifo<T>
 where
-    K: Key,
-    V: Value,
+    T: Send + Sync + 'static,
 {
-    type Handle = S3FifoHandle<K, V>;
+    type Item = S3FifoHandle<T>;
     type Config = S3FifoConfig;
 
     unsafe fn new(capacity: usize, config: &Self::Config) -> Self
@@ -211,7 +203,7 @@ where
         }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn push(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
 
         self.small_queue.push_back(ptr);
@@ -221,7 +213,7 @@ where
         handle.base_mut().set_in_eviction(true);
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+    unsafe fn pop(&mut self) -> Option<NonNull<Self::Item>> {
         if let Some(mut ptr) = self.evict() {
             let handle = ptr.as_mut();
             // `handle.queue` has already been set with `evict()`
@@ -233,14 +225,14 @@ where
         }
     }
 
-    unsafe fn reinsert(&mut self, _: NonNull<Self::Handle>) {}
+    unsafe fn reinsert(&mut self, _: NonNull<Self::Item>) {}
 
-    unsafe fn access(&mut self, ptr: NonNull<Self::Handle>) {
+    unsafe fn access(&mut self, ptr: NonNull<Self::Item>) {
         let mut ptr = ptr;
         ptr.as_mut().inc();
     }
 
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
+    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Item>) {
         let handle = ptr.as_mut();
 
         match handle.queue {
@@ -274,7 +266,7 @@ where
         }
     }
 
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
+    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Item>> {
         let mut res = Vec::with_capacity(self.len());
         while let Some(mut ptr) = self.small_queue.pop_front() {
             let handle = ptr.as_mut();
@@ -300,18 +292,8 @@ where
     }
 }
 
-unsafe impl<K, V> Send for S3Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
-unsafe impl<K, V> Sync for S3Fifo<K, V>
-where
-    K: Key,
-    V: Value,
-{
-}
+unsafe impl<T> Send for S3Fifo<T> where T: Send + Sync + 'static {}
+unsafe impl<T> Sync for S3Fifo<T> where T: Send + Sync + 'static {}
 
 #[cfg(test)]
 mod tests {
@@ -322,32 +304,24 @@ mod tests {
     use super::*;
     use crate::eviction::test_utils::TestEviction;
 
-    impl<K, V> TestEviction for S3Fifo<K, V>
+    impl<T> TestEviction for S3Fifo<T>
     where
-        K: Key + Clone,
-        V: Value + Clone,
+        T: Send + Sync + 'static + Clone,
     {
-        fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)> {
+        fn dump(&self) -> Vec<T> {
             self.small_queue
                 .iter()
                 .chain(self.main_queue.iter())
-                .map(|handle| (handle.base().key().clone(), handle.base().value().clone()))
+                .map(|handle| handle.base().data_unwrap_unchecked().clone())
                 .collect_vec()
         }
     }
 
-    type TestS3Fifo = S3Fifo<u64, u64>;
-    type TestS3FifoHandle = S3FifoHandle<u64, u64>;
+    type TestS3Fifo = S3Fifo<u64>;
+    type TestS3FifoHandle = S3FifoHandle<u64>;
 
     fn assert_test_s3fifo(s3fifo: &TestS3Fifo, small: Vec<u64>, main: Vec<u64>) {
-        let mut s = s3fifo
-            .dump()
-            .into_iter()
-            .map(|(k, v)| {
-                assert_eq!(k, v);
-                k
-            })
-            .collect_vec();
+        let mut s = s3fifo.dump().into_iter().collect_vec();
         assert_eq!(s.len(), s3fifo.small_queue.len() + s3fifo.main_queue.len());
         let m = s.split_off(s3fifo.small_queue.len());
         assert_eq!((&s, &m), (&small, &main));
@@ -366,7 +340,7 @@ mod tests {
             let ptrs = (0..100)
                 .map(|i| {
                     let mut handle = Box::new(TestS3FifoHandle::new());
-                    handle.init(i, i, i, 1, S3FifoContext);
+                    handle.init(i, i, 1, S3FifoContext);
                     NonNull::new_unchecked(Box::into_raw(handle))
                 })
                 .collect_vec();
diff --git a/foyer-memory/src/eviction/test_utils.rs b/foyer-memory/src/eviction/test_utils.rs
index adf478de..983ae8b9 100644
--- a/foyer-memory/src/eviction/test_utils.rs
+++ b/foyer-memory/src/eviction/test_utils.rs
@@ -17,6 +17,9 @@ use crate::handle::Handle;
 
 // TODO(MrCroxx): use `expect` after `lint_reasons` is stable.
 #[allow(clippy::type_complexity)]
-pub trait TestEviction: Eviction {
-    fn dump(&self) -> Vec<(<Self::Handle as Handle>::Key, <Self::Handle as Handle>::Value)>;
+pub trait TestEviction: Eviction
+where
+    Self::Item: Handle,
+{
+    fn dump(&self) -> Vec<<Self::Item as Handle>::Data>;
 }
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index d217fe12..9414f978 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -34,8 +34,12 @@ use parking_lot::Mutex;
 use tokio::{sync::oneshot, task::JoinHandle};
 
 use crate::{
-    eviction::Eviction, handle::Handle, indexer::Indexer, listener::CacheEventListener, metrics::Metrics, CacheContext,
-    Key, Value,
+    eviction::Eviction,
+    handle::{Handle, KeyedHandle},
+    indexer::Indexer,
+    listener::CacheEventListener,
+    metrics::Metrics,
+    CacheContext, Key, Value,
 };
 
 struct CacheSharedState<T, L> {
@@ -52,8 +56,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -65,7 +69,7 @@ where
 
     waiters: HashMap<K, Vec<oneshot::Sender<GenericCacheEntry<K, V, E, I, L, S>>>>,
 
-    state: Arc<CacheSharedState<E::Handle, L>>,
+    state: Arc<CacheSharedState<E::Item, L>>,
 }
 
 impl<K, V, E, I, L, S> CacheShard<K, V, E, I, L, S>
@@ -73,8 +77,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -82,7 +86,7 @@ where
         capacity: usize,
         eviction_config: &E::Config,
         usage: Arc<AtomicUsize>,
-        context: Arc<CacheSharedState<E::Handle, L>>,
+        context: Arc<CacheSharedState<E::Item, L>>,
     ) -> Self {
         let indexer = I::new();
         let eviction = unsafe { E::new(capacity, eviction_config) };
@@ -104,15 +108,11 @@ where
         key: K,
         value: V,
         charge: usize,
-        context: <E::Handle as Handle>::Context,
-        last_reference_entries: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>,
-    ) -> NonNull<E::Handle> {
-        let mut handle = self
-            .state
-            .object_pool
-            .pop()
-            .unwrap_or_else(|| Box::new(E::Handle::new()));
-        handle.init(hash, key, value, charge, context);
+        context: <E::Item as Handle>::Context,
+        last_reference_entries: &mut Vec<(K, V, <E::Item as Handle>::Context, usize)>,
+    ) -> NonNull<E::Item> {
+        let mut handle = self.state.object_pool.pop().unwrap_or_else(|| Box::new(E::Item::new()));
+        handle.init(hash, (key, value), charge, context);
         let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
 
         self.evict(charge, last_reference_entries);
@@ -144,7 +144,7 @@ where
         ptr
     }
 
-    unsafe fn get<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<E::Handle>>
+    unsafe fn get<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<E::Item>>
     where
         K: Borrow<Q>,
         Q: Hash + Eq + ?Sized,
@@ -179,7 +179,7 @@ where
     /// Remove a key from the cache.
     ///
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<(K, V, <E::Handle as Handle>::Context, usize)>
+    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<(K, V, <E::Item as Handle>::Context, usize)>
     where
         K: Borrow<Q>,
         Q: Hash + Eq + ?Sized,
@@ -195,7 +195,7 @@ where
     }
 
     /// Clear all cache entries.
-    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>) {
+    unsafe fn clear(&mut self, last_reference_entries: &mut Vec<(K, V, <E::Item as Handle>::Context, usize)>) {
         // TODO(MrCroxx): Avoid collecting here?
         let ptrs = self.indexer.drain().collect_vec();
         let eptrs = self.eviction.clear();
@@ -223,7 +223,7 @@ where
     unsafe fn evict(
         &mut self,
         charge: usize,
-        last_reference_entries: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>,
+        last_reference_entries: &mut Vec<(K, V, <E::Item as Handle>::Context, usize)>,
     ) {
         // TODO(MrCroxx): Use `let_chains` here after it is stable.
         while self.usage.load(Ordering::Relaxed) + charge > self.capacity {
@@ -246,8 +246,8 @@ where
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
     unsafe fn try_release_external_handle(
         &mut self,
-        mut ptr: NonNull<E::Handle>,
-    ) -> Option<(K, V, <E::Handle as Handle>::Context, usize)> {
+        mut ptr: NonNull<E::Item>,
+    ) -> Option<(K, V, <E::Item as Handle>::Context, usize)> {
         ptr.as_mut().base_mut().dec_refs();
         self.try_release_handle(ptr, true)
     }
@@ -259,26 +259,26 @@ where
     /// Recycle it if possible.
     unsafe fn try_release_handle(
         &mut self,
-        mut ptr: NonNull<E::Handle>,
+        mut ptr: NonNull<E::Item>,
         reinsert: bool,
-    ) -> Option<(K, V, <E::Handle as Handle>::Context, usize)> {
-        let base = ptr.as_mut().base_mut();
+    ) -> Option<(K, V, <E::Item as Handle>::Context, usize)> {
+        let handle = ptr.as_mut();
 
-        if base.has_refs() {
+        if handle.base().has_refs() {
             return None;
         }
 
-        debug_assert!(base.is_inited());
-        debug_assert!(!base.has_refs());
+        debug_assert!(handle.base().is_inited());
+        debug_assert!(!handle.base().has_refs());
 
         // If the entry is not updated or removed from the cache, try to reinsert it or remove it from the indexer and
         // the eviction container.
-        if base.is_in_indexer() {
+        if handle.base().is_in_indexer() {
             // The usage is higher than the capacity means most handles are held externally,
             // the cache shard cannot release enough charges for the new inserted entries.
             // In this case, the reinsertion should be given up.
             if reinsert && self.usage.load(Ordering::Relaxed) <= self.capacity {
-                let was_in_eviction = base.is_in_eviction();
+                let was_in_eviction = handle.base().is_in_eviction();
                 self.eviction.reinsert(ptr);
                 if ptr.as_ref().base().is_in_eviction() {
                     if was_in_eviction {
@@ -289,26 +289,26 @@ where
             }
 
             // If the entry has not been reinserted, remove it from the indexer and the eviction container (if needed).
-            self.indexer.remove(base.hash(), base.key());
+            self.indexer.remove(handle.base().hash(), handle.key());
             if ptr.as_ref().base().is_in_eviction() {
                 self.eviction.remove(ptr);
             }
         }
 
         // Here the handle is neither in the indexer nor in the eviction container.
-        debug_assert!(!base.is_in_indexer());
-        debug_assert!(!base.is_in_eviction());
-        debug_assert!(!base.has_refs());
+        debug_assert!(!handle.base().is_in_indexer());
+        debug_assert!(!handle.base().is_in_eviction());
+        debug_assert!(!handle.base().has_refs());
 
         self.state.metrics.release.fetch_add(1, Ordering::Relaxed);
 
-        self.usage.fetch_sub(base.charge(), Ordering::Relaxed);
-        let entry = base.take();
+        self.usage.fetch_sub(handle.base().charge(), Ordering::Relaxed);
+        let ((key, value), context, charge) = handle.base_mut().take();
 
         let handle = Box::from_raw(ptr.as_ptr());
         let _ = self.state.object_pool.push(handle);
 
-        Some(entry)
+        Some((key, value, context, charge))
     }
 }
 
@@ -317,8 +317,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -327,10 +327,13 @@ where
     }
 }
 
-pub struct CacheConfig<E, L, S = RandomState>
+pub struct CacheConfig<K, V, E, L, S = RandomState>
 where
+    K: Key,
+    V: Value,
     E: Eviction,
-    L: CacheEventListener<<E::Handle as Handle>::Key, <E::Handle as Handle>::Value>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     pub capacity: usize,
@@ -348,8 +351,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
@@ -365,8 +368,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error,
@@ -381,8 +384,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
     ER: std::error::Error + From<oneshot::error::RecvError>,
@@ -409,8 +412,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -419,7 +422,7 @@ where
     capacity: usize,
     usages: Vec<Arc<AtomicUsize>>,
 
-    context: Arc<CacheSharedState<E::Handle, L>>,
+    context: Arc<CacheSharedState<E::Item, L>>,
 
     hash_builder: S,
 }
@@ -429,12 +432,12 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn new(config: CacheConfig<E, L, S>) -> Self {
+    pub fn new(config: CacheConfig<K, V, E, L, S>) -> Self {
         let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
         let context = Arc::new(CacheSharedState {
             metrics: Metrics::default(),
@@ -572,7 +575,7 @@ where
         &self.context.metrics
     }
 
-    unsafe fn try_release_external_handle(&self, ptr: NonNull<E::Handle>) {
+    unsafe fn try_release_external_handle(&self, ptr: NonNull<E::Item>) {
         let entry = {
             let base = ptr.as_ref().base();
             let mut shard = self.shards[base.hash() as usize % self.shards.len()].lock();
@@ -592,8 +595,8 @@ where
     K: Key + Clone,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -653,13 +656,13 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
     cache: Arc<GenericCache<K, V, E, I, L, S>>,
-    ptr: NonNull<E::Handle>,
+    ptr: NonNull<E::Item>,
 }
 
 impl<K, V, E, I, L, S> GenericCacheEntry<K, V, E, I, L, S>
@@ -667,20 +670,20 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
-    pub fn key(&self) -> &<E::Handle as Handle>::Key {
-        unsafe { self.ptr.as_ref().base().key() }
+    pub fn key(&self) -> &K {
+        unsafe { &self.ptr.as_ref().base().data_unwrap_unchecked().0 }
     }
 
-    pub fn value(&self) -> &<E::Handle as Handle>::Value {
-        unsafe { self.ptr.as_ref().base().value() }
+    pub fn value(&self) -> &V {
+        unsafe { &self.ptr.as_ref().base().data_unwrap_unchecked().1 }
     }
 
-    pub fn context(&self) -> &<E::Handle as Handle>::Context {
+    pub fn context(&self) -> &<E::Item as Handle>::Context {
         unsafe { self.ptr.as_ref().base().context() }
     }
 
@@ -698,8 +701,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -724,8 +727,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -739,8 +742,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -756,8 +759,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -767,8 +770,8 @@ where
     K: Key,
     V: Value,
     E: Eviction,
-    E::Handle: Handle<Key = K, Value = V>,
-    I: Indexer<Key = K, Handle = E::Handle>,
+    E::Item: KeyedHandle<Key = K, Data = (K, V)>,
+    I: Indexer<Key = K, Handle = E::Item>,
     L: CacheEventListener<K, V>,
     S: BuildHasher + Send + Sync + 'static,
 {
@@ -864,7 +867,7 @@ mod tests {
     fn test_reference_count() {
         let cache = fifo(100);
 
-        let refs = |ptr: NonNull<FifoHandle<u64, String>>| unsafe { ptr.as_ref().base().refs() };
+        let refs = |ptr: NonNull<FifoHandle<(u64, String)>>| unsafe { ptr.as_ref().base().refs() };
 
         let e1 = insert_fifo(&cache, 42, "the answer to life, the universe, and everything");
         let ptr = e1.ptr;
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
index 602b4bf7..9e81e86e 100644
--- a/foyer-memory/src/handle.rs
+++ b/foyer-memory/src/handle.rs
@@ -25,25 +25,39 @@ bitflags! {
 }
 
 pub trait Handle: Send + Sync + 'static {
-    type Key: Key;
-    type Value: Value;
+    type Data;
     type Context: Context;
 
     fn new() -> Self;
-    fn init(&mut self, hash: u64, key: Self::Key, value: Self::Value, charge: usize, context: Self::Context);
+    fn init(&mut self, hash: u64, data: Self::Data, charge: usize, context: Self::Context);
 
-    fn base(&self) -> &BaseHandle<Self::Key, Self::Value, Self::Context>;
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Key, Self::Value, Self::Context>;
+    fn base(&self) -> &BaseHandle<Self::Data, Self::Context>;
+    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context>;
 }
 
-#[derive(Debug)]
-pub struct BaseHandle<K, V, C>
+pub trait KeyedHandle: Handle {
+    type Key;
+
+    fn key(&self) -> &Self::Key;
+}
+
+impl<K, V, T> KeyedHandle for T
 where
     K: Key,
     V: Value,
+    T: Handle<Data = (K, V)>,
 {
+    type Key = K;
+
+    fn key(&self) -> &Self::Key {
+        &self.base().data_unwrap_unchecked().0
+    }
+}
+
+#[derive(Debug)]
+pub struct BaseHandle<T, C> {
     /// key, value, context
-    entry: Option<(K, V, C)>,
+    entry: Option<(T, C)>,
     /// key hash
     hash: u64,
     /// entry charge
@@ -54,21 +68,13 @@ where
     flags: BaseHandleFlags,
 }
 
-impl<K, V, C> Default for BaseHandle<K, V, C>
-where
-    K: Key,
-    V: Value,
-{
+impl<T, C> Default for BaseHandle<T, C> {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl<K, V, C> BaseHandle<K, V, C>
-where
-    K: Key,
-    V: Value,
-{
+impl<T, C> BaseHandle<T, C> {
     /// Create a uninited handle.
     #[inline(always)]
     pub fn new() -> Self {
@@ -83,10 +89,10 @@ where
 
     /// Init handle with args.
     #[inline(always)]
-    pub fn init(&mut self, hash: u64, key: K, value: V, charge: usize, context: C) {
+    pub fn init(&mut self, hash: u64, data: T, charge: usize, context: C) {
         debug_assert!(self.entry.is_none());
         self.hash = hash;
-        self.entry = Some((key, value, context));
+        self.entry = Some((data, context));
         self.charge = charge;
         self.refs = 0;
         self.flags = BaseHandleFlags::empty();
@@ -94,12 +100,12 @@ where
 
     /// Take key and value from the handle and reset it to the uninited state.
     #[inline(always)]
-    pub fn take(&mut self) -> (K, V, C, usize) {
+    pub fn take(&mut self) -> (T, C, usize) {
         debug_assert!(self.entry.is_some());
         unsafe {
             self.entry
                 .take()
-                .map(|(key, value, context)| (key, value, context, self.charge))
+                .map(|(data, context)| (data, context, self.charge))
                 .unwrap_unchecked()
         }
     }
@@ -120,26 +126,15 @@ where
         self.hash
     }
 
-    /// Get key reference.
-    ///  
-    /// # Panics
+    /// Get data reference.
     ///
-    /// Panics if the handle is uninited.
-    #[inline(always)]
-    pub fn key(&self) -> &K {
-        debug_assert!(self.entry.is_some());
-        unsafe { self.entry.as_ref().map(|entry| &entry.0).unwrap_unchecked() }
-    }
-
-    /// Get value reference.
-    ///  
     /// # Panics
     ///
     /// Panics if the handle is uninited.
     #[inline(always)]
-    pub fn value(&self) -> &V {
+    pub fn data_unwrap_unchecked(&self) -> &T {
         debug_assert!(self.entry.is_some());
-        unsafe { self.entry.as_ref().map(|entry| &entry.1).unwrap_unchecked() }
+        unsafe { self.entry.as_ref().map(|entry| &entry.0).unwrap_unchecked() }
     }
 
     /// Get context reference.
@@ -150,7 +145,7 @@ where
     #[inline(always)]
     pub fn context(&self) -> &C {
         debug_assert!(self.entry.is_some());
-        unsafe { self.entry.as_ref().map(|entry| &entry.2).unwrap_unchecked() }
+        unsafe { self.entry.as_ref().map(|entry| &entry.1).unwrap_unchecked() }
     }
 
     /// Get the charge of the handle.
@@ -226,7 +221,7 @@ mod tests {
 
     #[test]
     fn test_base_handle_basic() {
-        let mut h = BaseHandle::<(), (), ()>::new();
+        let mut h = BaseHandle::<(), ()>::new();
         assert!(!h.is_in_indexer());
         assert!(!h.is_in_eviction());
 
diff --git a/foyer-memory/src/indexer.rs b/foyer-memory/src/indexer.rs
index 17e29a8d..0fbf2f38 100644
--- a/foyer-memory/src/indexer.rs
+++ b/foyer-memory/src/indexer.rs
@@ -16,11 +16,11 @@ use std::{borrow::Borrow, hash::Hash, ptr::NonNull};
 
 use hashbrown::hash_table::{Entry as HashTableEntry, HashTable};
 
-use crate::{handle::Handle, Key};
+use crate::{handle::KeyedHandle, Key};
 
 pub trait Indexer: Send + Sync + 'static {
     type Key: Key;
-    type Handle: Handle<Key = Self::Key>;
+    type Handle: KeyedHandle<Key = Self::Key>;
 
     fn new() -> Self;
     unsafe fn insert(&mut self, handle: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>>;
@@ -38,7 +38,7 @@ pub trait Indexer: Send + Sync + 'static {
 pub struct HashTableIndexer<K, H>
 where
     K: Key,
-    H: Handle<Key = K>,
+    H: KeyedHandle<Key = K>,
 {
     table: HashTable<NonNull<H>>,
 }
@@ -46,21 +46,21 @@ where
 unsafe impl<K, H> Send for HashTableIndexer<K, H>
 where
     K: Key,
-    H: Handle<Key = K>,
+    H: KeyedHandle<Key = K>,
 {
 }
 
 unsafe impl<K, H> Sync for HashTableIndexer<K, H>
 where
     K: Key,
-    H: Handle<Key = K>,
+    H: KeyedHandle<Key = K>,
 {
 }
 
 impl<K, H> Indexer for HashTableIndexer<K, H>
 where
     K: Key,
-    H: Handle<Key = K>,
+    H: KeyedHandle<Key = K>,
 {
     type Key = K;
     type Handle = H;
@@ -72,14 +72,14 @@ where
     }
 
     unsafe fn insert(&mut self, mut ptr: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>> {
-        let base = ptr.as_mut().base_mut();
+        let handle = ptr.as_mut();
 
-        debug_assert!(!base.is_in_indexer());
-        base.set_in_indexer(true);
+        debug_assert!(!handle.base().is_in_indexer());
+        handle.base_mut().set_in_indexer(true);
 
         match self.table.entry(
-            base.hash(),
-            |p| p.as_ref().base().key() == base.key(),
+            handle.base().hash(),
+            |p| p.as_ref().key() == handle.key(),
             |p| p.as_ref().base().hash(),
         ) {
             HashTableEntry::Occupied(mut o) => {
@@ -101,9 +101,7 @@ where
         Self::Key: Borrow<Q>,
         Q: Hash + Eq + ?Sized,
     {
-        self.table
-            .find(hash, |p| p.as_ref().base().key().borrow() == key)
-            .copied()
+        self.table.find(hash, |p| p.as_ref().key().borrow() == key).copied()
     }
 
     unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
@@ -111,11 +109,10 @@ where
         Self::Key: Borrow<Q>,
         Q: Hash + Eq + ?Sized,
     {
-        match self.table.entry(
-            hash,
-            |p| p.as_ref().base().key().borrow() == key,
-            |p| p.as_ref().base().hash(),
-        ) {
+        match self
+            .table
+            .entry(hash, |p| p.as_ref().key().borrow() == key, |p| p.as_ref().base().hash())
+        {
             HashTableEntry::Occupied(o) => {
                 let (mut p, _) = o.remove();
                 let b = p.as_mut().base_mut();

From 541a329dc74fc1b397003c1865f5b6a60464c46f Mon Sep 17 00:00:00 2001
From: xiaguan <751080330@qq.com>
Date: Sun, 14 Apr 2024 19:07:14 +0800
Subject: [PATCH 258/261] test: fuzz test for foyer-memory

---
 .gitignore                                   |   3 +
 Cargo.toml                                   |   1 +
 foyer-memory/fuzz/Cargo.toml                 |  21 ++++
 foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs | 116 +++++++++++++++++++
 foyer-workspace-hack/Cargo.toml              |   2 -
 5 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 foyer-memory/fuzz/Cargo.toml
 create mode 100644 foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs

diff --git a/.gitignore b/.gitignore
index 7e1c8dde..769a4793 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,9 @@
 
 /target
 
+foyer-memory/fuzz/artifacts/cache_fuzz/
+foyer-memory/fuzz/corpus/cache_fuzz
+
 Cargo.lock
 lcov.info
 
diff --git a/Cargo.toml b/Cargo.toml
index e11d5e22..dc165ffe 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
     "foyer-experimental-bench",
     "foyer-intrusive",
     "foyer-memory",
+    "foyer-memory/fuzz",
     "foyer-storage",
     "foyer-storage-bench",
     "foyer-workspace-hack",
diff --git a/foyer-memory/fuzz/Cargo.toml b/foyer-memory/fuzz/Cargo.toml
new file mode 100644
index 00000000..a7d348ea
--- /dev/null
+++ b/foyer-memory/fuzz/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "foyer-memory-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[dependencies]
+ahash = "0.8"
+foyer-memory = { path = ".." }
+foyer-workspace-hack = { version = "0.4", path = "../../foyer-workspace-hack" }
+libfuzzer-sys = "0.4"
+
+[package.metadata]
+cargo-fuzz = true
+
+[[bin]]
+name = "cache_fuzz"
+path = "fuzz_targets/cache_fuzz.rs"
+test = false
+doc = false
+bench = false
diff --git a/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs b/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
new file mode 100644
index 00000000..a52edb51
--- /dev/null
+++ b/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
@@ -0,0 +1,116 @@
+#![no_main]
+
+use std::sync::Arc;
+
+use ahash::RandomState;
+use libfuzzer_sys::fuzz_target;
+
+use foyer_memory::{
+    Cache, DefaultCacheEventListener, FifoCacheConfig, FifoConfig, LfuCacheConfig, LfuConfig, LruCacheConfig,
+    LruConfig, S3FifoCacheConfig, S3FifoConfig,
+};
+
+type CacheKey = u8;
+type CacheValue = u8;
+const SHARDS: usize = 1;
+const OBJECT_POOL_CAPACITY: usize = 16;
+
+fn new_fifo_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = FifoCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: FifoConfig {},
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::fifo(config))
+}
+
+fn new_lru_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = LruCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: LruConfig {
+            high_priority_pool_ratio: 0.1,
+        },
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::lru(config))
+}
+
+fn new_lfu_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = LfuCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: LfuConfig {
+            window_capacity_ratio: 0.1,
+            protected_capacity_ratio: 0.8,
+            cmsketch_eps: 0.001,
+            cmsketch_confidence: 0.9,
+        },
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::lfu(config))
+}
+
+fn new_s3fifo_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    let config = S3FifoCacheConfig {
+        capacity,
+        shards: SHARDS,
+        eviction_config: S3FifoConfig {
+            small_queue_capacity_ratio: 0.1,
+        },
+        object_pool_capacity: OBJECT_POOL_CAPACITY,
+        hash_builder: RandomState::default(),
+        event_listener: DefaultCacheEventListener::default(),
+    };
+
+    Arc::new(Cache::s3fifo(config))
+}
+
+fn create_cache(op: u8) -> Arc<Cache<CacheKey, CacheValue>> {
+    match op % 4 {
+        0 => new_fifo_cache(20),
+        1 => new_lru_cache(20),
+        2 => new_lfu_cache(20),
+        3 => new_s3fifo_cache(20),
+        _ => unreachable!(),
+    }
+}
+
+fuzz_target!(|data: &[u8]| {
+    if data.is_empty() {
+        return;
+    }
+    let cache = create_cache(data[0]);
+    let mut it = data.iter();
+    for &op in it.by_ref() {
+        match op % 4 {
+            0 => {
+                cache.insert(op, op, 1);
+            }
+            1 => match cache.get(&op) {
+                Some(v) => assert_eq!(*v, op),
+                None => {}
+            },
+            2 => {
+                cache.remove(&op);
+            }
+            3 => {
+                // Low probability to clear the cache
+                if op % 10 == 1 {
+                    cache.clear();
+                }
+            }
+            _ => unreachable!(),
+        }
+    }
+});
diff --git a/foyer-workspace-hack/Cargo.toml b/foyer-workspace-hack/Cargo.toml
index b03784d9..9ad39a48 100644
--- a/foyer-workspace-hack/Cargo.toml
+++ b/foyer-workspace-hack/Cargo.toml
@@ -30,7 +30,6 @@ futures-executor = { version = "0.3" }
 futures-sink = { version = "0.3" }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro", "channel", "io", "sink"] }
 hashbrown = { version = "0.14", features = ["raw"] }
-itertools = { version = "0.12" }
 libc = { version = "0.2", features = ["extra_traits"] }
 parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] }
 parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] }
@@ -42,7 +41,6 @@ tracing-core = { version = "0.1" }
 [build-dependencies]
 cc = { version = "1", default-features = false, features = ["parallel"] }
 either = { version = "1", default-features = false, features = ["use_std"] }
-itertools = { version = "0.12" }
 proc-macro2 = { version = "1" }
 quote = { version = "1" }
 syn = { version = "2", features = ["extra-traits", "full", "visit-mut"] }

From d5c435b014f93759036c0c3b1895bd2c803600d0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 14 Apr 2024 20:39:16 +0800
Subject: [PATCH 259/261] test: upload binaries when asan/lsan test fails
 (#335)

* test: upload binaries when asan/lsan test fails

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix if condition

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: use different name for different artifacts

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/template/template.yml      | 20 ++++++++++++++++++++
 .github/workflows/main.yml         | 20 ++++++++++++++++++++
 .github/workflows/pull-request.yml | 20 ++++++++++++++++++++
 3 files changed, 60 insertions(+)

diff --git a/.github/template/template.yml b/.github/template/template.yml
index 8164b0c7..ccad4ddf 100644
--- a/.github/template/template.yml
+++ b/.github/template/template.yml
@@ -164,6 +164,16 @@ jobs:
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+      - name: Prepare Artifacts on Failure
+        if: ${{ failure() }}
+        run: |-
+          find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.asan.tgz --transform 's#.*/##'
+      - name: Upload Artifacts on Failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: artifacts.asan.tgz
+          path: artifacts.asan.tgz
   lsan:
     name: run with leak saniziter
     runs-on: ubuntu-latest
@@ -200,6 +210,16 @@ jobs:
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+      - name: Prepare Artifacts on Failure
+        if: ${{ failure() }}
+        run: |-
+          find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.lsan.tgz --transform 's#.*/##'
+      - name: Upload Artifacts on Failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: artifacts.lsan.tgz
+          path: artifacts.lsan.tgz
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d9efce08..2e3bac35 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -171,6 +171,16 @@ jobs:
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+      - name: Prepare Artifacts on Failure
+        if: ${{ failure() }}
+        run: |-
+          find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.asan.tgz --transform 's#.*/##'
+      - name: Upload Artifacts on Failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: artifacts.asan.tgz
+          path: artifacts.asan.tgz
   lsan:
     name: run with leak saniziter
     runs-on: ubuntu-latest
@@ -207,6 +217,16 @@ jobs:
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+      - name: Prepare Artifacts on Failure
+        if: ${{ failure() }}
+        run: |-
+          find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.lsan.tgz --transform 's#.*/##'
+      - name: Upload Artifacts on Failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: artifacts.lsan.tgz
+          path: artifacts.lsan.tgz
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 3d73dacf..a8b0fd06 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -170,6 +170,16 @@ jobs:
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+      - name: Prepare Artifacts on Failure
+        if: ${{ failure() }}
+        run: |-
+          find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.asan.tgz --transform 's#.*/##'
+      - name: Upload Artifacts on Failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: artifacts.asan.tgz
+          path: artifacts.asan.tgz
   lsan:
     name: run with leak saniziter
     runs-on: ubuntu-latest
@@ -206,6 +216,16 @@ jobs:
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
           timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+      - name: Prepare Artifacts on Failure
+        if: ${{ failure() }}
+        run: |-
+          find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.lsan.tgz --transform 's#.*/##'
+      - name: Upload Artifacts on Failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: artifacts.lsan.tgz
+          path: artifacts.lsan.tgz
   deterministic-test:
     name: run deterministic test
     runs-on: ubuntu-latest

From 1f7f231abcae1977947aa462ed37276dea51fdfd Mon Sep 17 00:00:00 2001
From: xiaguan <751080330@qq.com>
Date: Sun, 14 Apr 2024 19:49:30 +0800
Subject: [PATCH 260/261] more readable fuzz test

Signed-off-by: xiaguan <751080330@qq.com>
---
 foyer-memory/fuzz/Cargo.toml                 |  1 +
 foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs | 73 ++++++++++++--------
 2 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/foyer-memory/fuzz/Cargo.toml b/foyer-memory/fuzz/Cargo.toml
index a7d348ea..da46b5ee 100644
--- a/foyer-memory/fuzz/Cargo.toml
+++ b/foyer-memory/fuzz/Cargo.toml
@@ -9,6 +9,7 @@ ahash = "0.8"
 foyer-memory = { path = ".." }
 foyer-workspace-hack = { version = "0.4", path = "../../foyer-workspace-hack" }
 libfuzzer-sys = "0.4"
+arbitrary = { version = "1", features = ["derive"] }
 
 [package.metadata]
 cargo-fuzz = true
diff --git a/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs b/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
index a52edb51..daf9988e 100644
--- a/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
+++ b/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
@@ -3,6 +3,7 @@
 use std::sync::Arc;
 
 use ahash::RandomState;
+use arbitrary::Arbitrary;
 use libfuzzer_sys::fuzz_target;
 
 use foyer_memory::{
@@ -76,41 +77,55 @@ fn new_s3fifo_cache(capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
     Arc::new(Cache::s3fifo(config))
 }
 
-fn create_cache(op: u8) -> Arc<Cache<CacheKey, CacheValue>> {
-    match op % 4 {
-        0 => new_fifo_cache(20),
-        1 => new_lru_cache(20),
-        2 => new_lfu_cache(20),
-        3 => new_s3fifo_cache(20),
-        _ => unreachable!(),
-    }
+#[derive(Debug, Arbitrary)]
+enum Op {
+    Insert(CacheKey, CacheValue, usize),
+    Get(CacheKey),
+    Remove(CacheKey),
+    Clear,
+}
+
+#[derive(Debug, Arbitrary)]
+enum CacheType {
+    Fifo,
+    Lru,
+    Lfu,
+    S3Fifo,
+}
+
+#[derive(Debug, Arbitrary)]
+struct Input {
+    capacity: usize,
+    cache_type: CacheType,
+    operations: Vec<Op>,
 }
 
-fuzz_target!(|data: &[u8]| {
-    if data.is_empty() {
-        return;
+fn create_cache(cache_type: CacheType, capacity: usize) -> Arc<Cache<CacheKey, CacheValue>> {
+    match cache_type {
+        CacheType::Fifo => new_fifo_cache(capacity),
+        CacheType::Lru => new_lru_cache(capacity),
+        CacheType::Lfu => new_lfu_cache(capacity),
+        CacheType::S3Fifo => new_s3fifo_cache(capacity),
     }
-    let cache = create_cache(data[0]);
-    let mut it = data.iter();
-    for &op in it.by_ref() {
-        match op % 4 {
-            0 => {
-                cache.insert(op, op, 1);
+}
+
+fuzz_target!(|data: Input| {
+    let cache = create_cache(data.cache_type, data.capacity);
+
+    for op in data.operations {
+        match op {
+            Op::Insert(k, v, size) => {
+                cache.insert(k, v, size as usize);
+            }
+            Op::Get(k) => {
+                let _ = cache.get(&k);
             }
-            1 => match cache.get(&op) {
-                Some(v) => assert_eq!(*v, op),
-                None => {}
-            },
-            2 => {
-                cache.remove(&op);
+            Op::Remove(k) => {
+                cache.remove(&k);
             }
-            3 => {
-                // Low probability to clear the cache
-                if op % 10 == 1 {
-                    cache.clear();
-                }
+            Op::Clear => {
+                cache.clear();
             }
-            _ => unreachable!(),
         }
     }
 });

From 75c8a510385a91a1582afede72563df75ff77cdc Mon Sep 17 00:00:00 2001
From: xiaguan <751080330@qq.com>
Date: Sun, 14 Apr 2024 22:27:00 +0800
Subject: [PATCH 261/261] add license

Signed-off-by: xiaguan <751080330@qq.com>
---
 foyer-memory/fuzz/Cargo.toml                 |  2 +-
 foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/foyer-memory/fuzz/Cargo.toml b/foyer-memory/fuzz/Cargo.toml
index da46b5ee..206453d6 100644
--- a/foyer-memory/fuzz/Cargo.toml
+++ b/foyer-memory/fuzz/Cargo.toml
@@ -6,10 +6,10 @@ edition = "2021"
 
 [dependencies]
 ahash = "0.8"
+arbitrary = { version = "1", features = ["derive"] }
 foyer-memory = { path = ".." }
 foyer-workspace-hack = { version = "0.4", path = "../../foyer-workspace-hack" }
 libfuzzer-sys = "0.4"
-arbitrary = { version = "1", features = ["derive"] }
 
 [package.metadata]
 cargo-fuzz = true
diff --git a/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs b/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
index daf9988e..f3c47405 100644
--- a/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
+++ b/foyer-memory/fuzz/fuzz_targets/cache_fuzz.rs
@@ -1,3 +1,17 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
 #![no_main]
 
 use std::sync::Arc;
@@ -115,7 +129,7 @@ fuzz_target!(|data: Input| {
     for op in data.operations {
         match op {
             Op::Insert(k, v, size) => {
-                cache.insert(k, v, size as usize);
+                cache.insert(k, v, size);
             }
             Op::Get(k) => {
                 let _ = cache.get(&k);