diff --git a/cabal.project b/cabal.project index 25874410b4e..ed2c57a5919 100644 --- a/cabal.project +++ b/cabal.project @@ -31,6 +31,7 @@ packages: trace-dispatcher trace-resources trace-forward + timeseries-io -- Needed when cross compiling extra-packages: alex diff --git a/timeseries-io/CHANGELOG.md b/timeseries-io/CHANGELOG.md new file mode 100644 index 00000000000..07fb028f500 --- /dev/null +++ b/timeseries-io/CHANGELOG.md @@ -0,0 +1,5 @@ +# Revision history for timeseries-io + +## 0.1.0.0 -- YYYY-mm-dd + +* First version. Released on an unsuspecting world. diff --git a/timeseries-io/app/Main.hs b/timeseries-io/app/Main.hs new file mode 100644 index 00000000000..fb120ceb500 --- /dev/null +++ b/timeseries-io/app/Main.hs @@ -0,0 +1,67 @@ +{-# OPTIONS_GHC -Wno-deprecations #-} +{-# OPTIONS_GHC -Wno-name-shadowing #-} + +module Main where + +import Cardano.Timeseries.Import.PlainCBOR +import Cardano.Timeseries.Query (interp) +import Cardano.Timeseries.Query.Parser (expr) +import Cardano.Timeseries.Query.Value (Error, Value) +import Cardano.Timeseries.Store +import Cardano.Timeseries.Store.Flat (Flat, Point (instant, name)) +import Cardano.Timeseries.Store.Parser (points) + +import Control.Monad (forever) +import Control.Monad.Except (runExceptT) +import Control.Monad.State.Strict (evalState) +import Data.Attoparsec (skipMany) +import Data.Attoparsec.Text (decimal, endOfInput, parseOnly, space) +import Data.Foldable (for_, traverse_) +import Data.Text (pack) +import GHC.List (foldl') +import System.Exit (die) +import System.IO (hFlush, stdout) +import Cardano.Timeseries.Store.Tree (fromFlat) +import Cardano.Logging.Resources (readResourceStats) +import Cardano.Logging (forHuman) +import Cardano.Logging.Resources (ResourceStats) +import Cardano.Logging.Resources (Resources(..)) + +snapshotsFile :: String +snapshotsFile = "6nodes_4hours_1mininterval.cbor" + +printStore :: Flat Double -> IO () +printStore = traverse_ print + +printQueryResult :: Either Error Value -> IO () +printQueryResult (Left err) = putStrLn ("Error: " <> err) +printQueryResult (Right ok) = print ok + +printStats :: ResourceStats -> IO () +printStats stats = + putStrLn $ "Alloc: " <> show ((fromIntegral (rAlloc stats) :: Double) / 1024 / 1024) <> "MB\n" + <> "Live: " <> show ((fromIntegral (rLive stats) :: Double) / 1024 / 1024) <> "MB\n" + <> "Heap: " <> show ((fromIntegral (rHeap stats) :: Double) / 1024 / 1024) <> "MB\n" + <> "RSS: " <> show ((fromIntegral (rRSS stats) :: Double) / 1024 / 1024) <> "MB" + +interactive :: Store s Double => s -> IO () +interactive store = forever $ do + Just stats <- readResourceStats + putStrLn "----------" + printStats stats + putStrLn $ "Number of store entries: " <> show (count store) + putStrLn "----------" + putStr "> " + hFlush stdout + queryString <- getLine + case parseOnly (expr <* skipMany space <* endOfInput) (pack queryString) of + Left err -> putStrLn err + Right query -> do + putStrLn ("Expr: " <> show query) + printQueryResult (evalState (runExceptT $ interp store mempty query 0) 0) + +main :: IO () +main = do + content <- readFileSnapshots snapshotsFile + let store = snapshotsToFlatStore content + interactive (fromFlat store) diff --git a/timeseries-io/bench/Bench.hs b/timeseries-io/bench/Bench.hs new file mode 100644 index 00000000000..8a134f6ebd9 --- /dev/null +++ b/timeseries-io/bench/Bench.hs @@ -0,0 +1,62 @@ +{-# OPTIONS_GHC -Wno-name-shadowing #-} +{-# OPTIONS_GHC -Wno-incomplete-uni-patterns #-} +{-# OPTIONS_GHC -Wno-deprecations #-} +import Cardano.Logging (forHuman) +import Cardano.Logging.Resources (ResourceStats, Resources (..), readResourceStats) +import Cardano.Timeseries.Import.PlainCBOR +import Cardano.Timeseries.Query (interp) +import Cardano.Timeseries.Query.Expr (Expr) +import Cardano.Timeseries.Query.Parser (expr) +import Cardano.Timeseries.Query.Value (Error, Value) +import Cardano.Timeseries.Store +import Cardano.Timeseries.Store.Flat (Flat, Point (instant, name)) +import Cardano.Timeseries.Store.Parser (points) +import Cardano.Timeseries.Store.Tree (fromFlat) + +import Control.Monad (forever) +import Control.Monad.Except (runExceptT) +import Control.Monad.State.Strict (evalState) +import Data.Attoparsec (skipMany) +import Data.Attoparsec.Text (decimal, endOfInput, parseOnly, space) +import Data.Either (fromRight) +import Data.Foldable (for_, traverse_) +import Data.Text (pack) +import GHC.List (foldl') +import System.Exit (die) +import System.IO (hFlush, stdout) + +import Criterion.Main + +-- Given a snapshots file +-- Given a query string +-- Bench mark evaluation of the query + +snapshotsFile :: String +snapshotsFile = "data/6nodes_4hours_1mininterval.cbor" + +query :: String +query = "\ + \let start = milliseconds 1762173870000 in \ + \let period = minutes 10 in \ + \let F = range Forge_forged_counter \ + \(fast_forward epoch start) \ + \(fast_forward epoch (add_duration start period)) in \ + \increase F" + +action :: Store s Double => (s, Expr) -> Value +action (store, query) = + let Right !x = evalState (runExceptT $ interp store mempty query 0) 0 in x + +main :: IO () +main = do + content <- readFileSnapshots snapshotsFile + let flatStore = snapshotsToFlatStore content + let treeStore = fromFlat flatStore + case parseOnly (expr <* skipMany space <* endOfInput) (pack query) of + Left err -> putStrLn err + Right !query -> defaultMain + [ + bench "flat" $ nf action (flatStore, query), + bench "tree" $ nf action (treeStore, query) + ] + diff --git a/timeseries-io/data/6nodes_4hours_1mininterval.cbor b/timeseries-io/data/6nodes_4hours_1mininterval.cbor new file mode 100644 index 00000000000..69d7314c597 Binary files /dev/null and b/timeseries-io/data/6nodes_4hours_1mininterval.cbor differ diff --git a/timeseries-io/doc.txt b/timeseries-io/doc.txt new file mode 100644 index 00000000000..3df20da7f09 --- /dev/null +++ b/timeseries-io/doc.txt @@ -0,0 +1,19 @@ +An instant of type `a` is the following data: +— A set of labels +— A timestamp +— An element of `a` +The set of labels identifies which timeseries the instant belongs to. +We sometimes call this set of labels a "series identifier". + +An instant vector of type `a` is a set of `instant`s of type `a` where every two elements of the set have distinct: +— Set of labels + +A timeseries of type `a` is: +— A set of labels +— a set of pairs of: + — A timestamp + — An element of `a` + +A timeseries vector of type `a` is a set of timeseries where every two elements of the set have distinct: +— Set of labels + diff --git a/timeseries-io/elab.txt b/timeseries-io/elab.txt new file mode 100644 index 00000000000..dcfe1700317 --- /dev/null +++ b/timeseries-io/elab.txt @@ -0,0 +1,108 @@ +// WIP: In case we want some ambiguity in the surface language + +fst t +snd t +(t, t) +t == t +t < t +t <= t +t > t +t >= t +t + t +t - t +t * t +t / t +not a +a && b +a || b +\x -> t +minutes n +hours n +epoch +now +rewind t d +fast_forward t d +toScalar t +s[t; t] +s[t; t; t] +v{ls} +max t +avg t +filter t t +t ⊗ t +t t +abs t : Scalar +increase t +rate t +avg_over_time t +sum_over_time t +quantile_over_time t t +unless t t +quantile-by ls t t + +Γ |- t ~> t' ∈ (A, B) +------------------------ +Γ |- fst t ~> fst t' ∈ A + + +Γ |- t ~> t' ∈ (A, B) +------------------- +Γ |- snd t ~> snd t' ∈ B + + +Γ |- a ~> a' ∈ A +Γ |- b ~> b' ∈ B +-------------------------------- +Γ |- (a, b) ~> (a', b') ∈ (A, B) + + +Γ |- a₀ ~> a₀' ∈ A +Γ |- a₁ ~> a₁' ∈ B +Γ |- A = B = Scalar +------------------------------------ +Γ |- a₀ == a₁ ~> eq_scalar a₀' a₁' ∈ Scalar + + +Γ |- a₀ ~> a₀' ∈ A +Γ |- a₁ ~> a₁' ∈ B +Γ |- A = B = Bool +--------------------------------------- +Γ |- a₀ == a₁ ~> eq_bool a₀' a₁' ∈ Bool + + +Γ |- a₀ ~> a₀' ∈ Scalar +Γ |- a₁ ~> a₁' ∈ Scalar +----------------------------------------- +Γ |- a₀ < a₁ ~> lt_scalar a₀' a₁' ∈ Scalar + + +Γ |- a₀ ~> a₀' ∈ InstantVector Scalar +Γ |- a₁ ~> a₁' ∈ Scalar +----------------------------------------------------------------------- +Γ |- a₀ < a₁ ~> lt_instant_vector_scalar a₀' a₁' ∈ InstantVector Scalar + + +Γ |- a₀ ~> a₀' ∈ Scalar +Γ |- a₁ ~> a₁' ∈ Scalar +---------------------------------------- +Γ |- a₀ + a₁ ~> add_scalar a₀ a₁ ∈ Scalar + + +Γ |- a₀ ~> a₀' ∈ InstantVector Scalar +Γ |- a₁ ~> a₁' ∈ InstantVector Scalar +---------------------------------------------------------------------- +Γ |- a₀ + a₁ ~> add_instant_vector_scalar a₀ a₁ ∈ InstantVector Scalar + + +Γ |- v ∋ Timestamp -> InstantVector Scalar +Γ |- a ∋ Timestamp +Γ |- b ∋ Timestamp +------------------------------------------ +Γ |- v[a, b] ∈ RangeVector Scalar + +Γ |- v ∋ Timestamp -> InstantVector Scalar +Γ |- a ∋ Timestamp +Γ |- b ∋ Timestamp +Γ |- d ∋ Duration +------------------------------------------ +Γ |- v[a, b, d] ∈ RangeVector Scalar diff --git a/timeseries-io/src/Cardano/Timeseries/Domain/Identifier.hs b/timeseries-io/src/Cardano/Timeseries/Domain/Identifier.hs new file mode 100644 index 00000000000..77ae2d9cdef --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Domain/Identifier.hs @@ -0,0 +1,6 @@ +module Cardano.Timeseries.Domain.Identifier(Identifier(..)) where + +-- | Identifiers come in two sorts: Userspace and Machine-generated. +-- | The first kind comes from user-typed expressions. +-- | The second kind is used for automatic code-generation for hygienic scoping (i.e. to avoid unintentional variable capture) +data Identifier = User String | Machine Int deriving (Show, Ord, Eq) diff --git a/timeseries-io/src/Cardano/Timeseries/Domain/Instant.hs b/timeseries-io/src/Cardano/Timeseries/Domain/Instant.hs new file mode 100644 index 00000000000..444888f3b85 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Domain/Instant.hs @@ -0,0 +1,38 @@ +module Cardano.Timeseries.Domain.Instant(Instant(..), InstantVector, mostRecent, share, toVector, prettyInstant, prettyInstantVector) where + +import Cardano.Timeseries.Domain.Types (SeriesIdentifier, Timestamp) + +import Control.DeepSeq (NFData) +import qualified Data.Set as Set +import Data.Text (Text, intercalate, pack) +import Data.Vector +import GHC.Generics (Generic) + +-- | One datapoint in a series. +data Instant a = Instant { + labels :: SeriesIdentifier, + timestamp :: Timestamp, + value :: a +} deriving (Show, Eq, Functor, Foldable, Traversable, Generic) + +instance NFData a => NFData (Instant a) + +-- | Do the instant vectors share a series? +share :: Instant a -> Instant b -> Bool +share a b = labels a == labels b + +-- | Datapoints from different series. The vector must not contain datapoints sharing a series. +type InstantVector a = [Instant a] + +mostRecent :: Instant a -> Instant a -> Instant a +mostRecent u v = if timestamp u < timestamp v then v else u + +toVector :: InstantVector Double -> Vector Double +toVector = fromList . fmap value + +prettyInstant :: Show a => Instant a -> Text +prettyInstant (Instant ls t v) = + pack (show (Set.toList ls)) <> " " <> pack (show t) <> " " <> pack (show v) + +prettyInstantVector :: Show a => InstantVector a -> Text +prettyInstantVector = intercalate "\n" . fmap prettyInstant diff --git a/timeseries-io/src/Cardano/Timeseries/Domain/Interval.hs b/timeseries-io/src/Cardano/Timeseries/Domain/Interval.hs new file mode 100644 index 00000000000..26f8ff6656e --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Domain/Interval.hs @@ -0,0 +1,14 @@ +module Cardano.Timeseries.Domain.Interval(Interval(..), length) where + +import Cardano.Timeseries.Domain.Types (Timestamp) + +import Prelude hiding (length) + +-- | A time interval. Assumption: `start` ≤ `end` +data Interval = Interval { + start :: Timestamp, + end :: Timestamp +} deriving (Show, Eq) + +length :: Interval -> Double +length (Interval s e) = fromIntegral (e - s) / 2 diff --git a/timeseries-io/src/Cardano/Timeseries/Domain/Timeseries.hs b/timeseries-io/src/Cardano/Timeseries/Domain/Timeseries.hs new file mode 100644 index 00000000000..9ed4f25974d --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Domain/Timeseries.hs @@ -0,0 +1,97 @@ +{-# OPTIONS_GHC -Wno-name-shadowing #-} +{-# LANGUAGE RecordWildCards #-} + +module Cardano.Timeseries.Domain.Timeseries(Timeseries(..), TimeseriesVector, + transpose, toVector, oldest, newest, eachOldest, eachNewest) where + +import Cardano.Timeseries.Domain.Instant (Instant (Instant), InstantVector) +import qualified Cardano.Timeseries.Domain.Instant as Instant +import Cardano.Timeseries.Domain.Types + +import Control.DeepSeq (NFData) +import Data.List (find, maximumBy, minimumBy) +import Data.Set +import qualified Data.Set as Set +import Data.Vector (Vector) +import qualified Data.Vector as Vector +import GHC.Generics (Generic) + +-- | A collection of datapoints sharing a series. +data Timeseries a = Timeseries { + labels :: SeriesIdentifier, + dat :: [(Timestamp, a)] +} deriving (Show, Functor, Foldable, Traversable, Generic) + +instance NFData a => NFData (Timeseries a) + +oldest :: Timeseries a -> Maybe (Instant a) +oldest Timeseries{..} | Prelude.null dat = Nothing +oldest Timeseries{..} = + let (t, x) = minimumBy (\(x, _) (y, _) -> compare x y) dat in + Just (Instant labels t x) + +newest :: Timeseries a -> Maybe (Instant a) +newest Timeseries{..} | Prelude.null dat = Nothing +newest Timeseries{..} = + let (t, x) = maximumBy (\(x, _) (y, _) -> compare x y) dat in + Just (Instant labels t x) + +-- | Every two elements in the list must have distinct series identifiers (set of labels), +-- | i.e. the series in the list must be distinct. +type TimeseriesVector a = [Timeseries a] + +eachOldest :: TimeseriesVector a -> Maybe [Instant a] +eachOldest = traverse oldest + +eachNewest :: TimeseriesVector a -> Maybe [Instant a] +eachNewest = traverse newest + +-- | Given a list of range vectors, forms up a timeseries vector. +-- | This operation is, in some sense, transposition: +-- | +-- | ⎴ ⎴ ⎴ ⎴ +-- | series1: ... ◯ ◯ ... +-- | series2: ... ◯ ◯ ... +-- | series3: ... ◯ ◯ ◯ ... +-- | ... ... +-- | ⎵ ⎵ ⎵ ⎵ +-- | -------------------------------> t +-- | t₀ t₁ t₂ t₃ +-- | +-- | =====> +-- | +-- | +-- | +-- | series1: [ ... ◯ ◯ ... ] +-- | series2: [ ... ◯ ◯ ... ] +-- | series3: [ ... ◯ ◯ ◯ ... ] +-- | ... ... +-- | +-- | ----------------------------------------> t +-- t₀ t₁ t₂ t₃ +transpose :: [InstantVector a] -> TimeseriesVector a +transpose instants = + Set.foldl' (\vector ls -> form ls instants : vector) [] (setOfLabels instants) where + + -- | Given a set of labels (identifying a series) form up a series from a list of instant vectors. + form :: SeriesIdentifier -> [InstantVector a] -> Timeseries a + form ls insts = Timeseries ls (form ls insts) where + -- | Extract the data pertaining to the series (identified by the given `SeriesIdentifier`) from the list of + -- | ranges vectors. + form :: SeriesIdentifier -> [InstantVector a] -> [(Timestamp, a)] + form _ [] = [] + form ls (inst : insts) = + case find (\i -> Instant.labels i == ls) inst of + Just i -> (Instant.timestamp i, Instant.value i) : form ls insts + Nothing -> form ls insts + + setOfLabels' :: InstantVector a -> Set SeriesIdentifier + setOfLabels' [] = Set.empty + setOfLabels' (i : is) = Set.insert (Instant.labels i) (setOfLabels' is) + + setOfLabels :: [InstantVector a] -> Set SeriesIdentifier + setOfLabels [] = Set.empty + setOfLabels (v : vs) = setOfLabels' v `Set.union` setOfLabels vs + +toVector :: Timeseries Double -> Vector Double +toVector = Vector.fromList . fmap snd . dat diff --git a/timeseries-io/src/Cardano/Timeseries/Domain/Types.hs b/timeseries-io/src/Cardano/Timeseries/Domain/Types.hs new file mode 100644 index 00000000000..7890f16c29a --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Domain/Types.hs @@ -0,0 +1,19 @@ +module Cardano.Timeseries.Domain.Types(MetricIdentifier, Label, Labelled, Timestamp, SeriesIdentifier) where + +import Prelude hiding (length) + +import Data.Set (Set) +import Data.Word (Word64) + +-- | Each series in the (metric) store can be identified by a metric name. +type MetricIdentifier = String + +type Label = String + +-- | Key-value pair of a label and its value. +type Labelled a = (Label, a) + +-- | Series is identified by a set of labels. Hence the name. +type SeriesIdentifier = Set (Labelled String) + +type Timestamp = Word64 diff --git a/timeseries-io/src/Cardano/Timeseries/Import/PlainCBOR.hs b/timeseries-io/src/Cardano/Timeseries/Import/PlainCBOR.hs new file mode 100644 index 00000000000..c2e1f673f87 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Import/PlainCBOR.hs @@ -0,0 +1,66 @@ +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE TypeSynonymInstances #-} + +{-# OPTIONS_GHC -Wno-type-defaults -Wno-orphans #-} + +module Cardano.Timeseries.Import.PlainCBOR where + +import Cardano.Timeseries.Domain.Instant (Instant (..)) +import Cardano.Timeseries.Store.Flat (Flat, Point (..)) + +import Codec.Serialise +import Control.Applicative +import Control.Monad (join) +import Data.Map.Strict as Map (Map, size) +import qualified Data.Map.Strict as Map +import qualified Data.Set as Set +import Data.Text (Text, unpack) +import Data.Time.Clock.POSIX (POSIXTime) +import GHC.Generics (Generic) + + +data NumericValue = + NVInt Int + | NVDouble Double + deriving (Generic, Show, Serialise) + + +data Snapshot = Snapshot + { singletonLabel :: Text + , timeStamp :: POSIXTime + , scrape :: Map Text NumericValue + } + deriving (Generic, Serialise) + +instance Show Snapshot where + show (Snapshot l t s) = "Snapshot{" ++ unpack l ++ "} @ " ++ show t ++ ", entries: " ++ show s + + +instance Serialise POSIXTime where + encode = encode . toInteger . floor + decode = fromInteger <$> decode + + +readFileSnapshots :: FilePath -> IO [Snapshot] +readFileSnapshots = readFileDeserialise + +numericValueToDouble :: NumericValue -> Double +numericValueToDouble (NVInt x) = fromIntegral x +numericValueToDouble (NVDouble x) = x + +scrapeDatapointToPoint :: Text -> POSIXTime -> Text -> NumericValue -> Point Double +scrapeDatapointToPoint node t metric v = + Point (unpack metric) (Instant (Set.fromList [("node", unpack node)]) (floor (t * 1000)) (numericValueToDouble v)) + +snapshotToFlatStore :: Snapshot -> Flat Double +snapshotToFlatStore (Snapshot l t s) = Map.foldlWithKey' (\acc k v -> scrapeDatapointToPoint l t k v : acc) [] s + +snapshotsToFlatStore :: [Snapshot] -> Flat Double +snapshotsToFlatStore = (>>= snapshotToFlatStore) + +-- can be used with Data.List.sortBy +snapshotOrd :: Snapshot -> Snapshot -> Ordering +snapshotOrd a b = + singletonLabel a `compare` singletonLabel b + <> timeStamp a `compare` timeStamp b diff --git a/timeseries-io/src/Cardano/Timeseries/Query.hs b/timeseries-io/src/Cardano/Timeseries/Query.hs new file mode 100644 index 00000000000..bb8f1e706ba --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Query.hs @@ -0,0 +1,388 @@ +{-# OPTIONS_GHC -Wno-name-shadowing #-} +{-# OPTIONS_GHC -Wno-typed-holes #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE ViewPatterns #-} +module Cardano.Timeseries.Query(interp) where +import Cardano.Timeseries.Domain.Identifier (Identifier (..)) +import Cardano.Timeseries.Domain.Instant (Instant (Instant), InstantVector, share) +import qualified Cardano.Timeseries.Domain.Instant as Domain +import qualified Cardano.Timeseries.Domain.Instant as Instant +import Cardano.Timeseries.Domain.Interval +import Cardano.Timeseries.Domain.Timeseries (Timeseries (Timeseries), TimeseriesVector, + eachNewest, eachOldest, transpose) +import qualified Cardano.Timeseries.Domain.Timeseries as Timeseries +import Cardano.Timeseries.Domain.Types (Labelled, MetricIdentifier, Timestamp) +import Cardano.Timeseries.Query.BinaryRelation (BinaryRelation, embedScalar, + mbBinaryRelationInstantVector, mbBinaryRelationScalar) +import qualified Cardano.Timeseries.Query.BinaryRelation as BinaryRelation +import Cardano.Timeseries.Query.Expr as Expr +import Cardano.Timeseries.Query.Value as Value +import Cardano.Timeseries.Store (Store (metrics)) +import qualified Cardano.Timeseries.Store as Store +import Cardano.Timeseries.Util (maybeToEither, safeToDouble, safeToWord64) + +import Control.Monad (filterM, (<=<)) +import Control.Monad.Except (ExceptT, liftEither, throwError) +import Control.Monad.State (get, put) +import Control.Monad.State.Strict (State) +import Control.Monad.Trans (lift) +import Data.List (find) +import Data.List.NonEmpty (fromList, toList) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromJust) +import Data.Set (isSubsetOf, member) +import qualified Data.Set as Set +import Data.Word (Word64) +import GHC.Base (NonEmpty ((:|))) + +import Statistics.Function (minMax) +import Statistics.Quantile (cadpw, quantile) +import Statistics.Sample (mean) + +join :: (a -> b -> c) -> InstantVector a -> InstantVector b -> Either Error (InstantVector c) +join _ [] _ = Right [] +join f (inst@(Domain.Instant ls t v) : xs) other = do + Domain.Instant _ _ v' <- maybeToEither ("No matching label: " <> show ls) $ find (share inst) other + rest <- join f xs other + Right (Domain.Instant ls t (f v v') : rest) + +avgOverTime :: Timestamp -> TimeseriesVector Double -> InstantVector Double +avgOverTime at = fmap compute where + compute :: Timeseries Double -> Instant Double + compute series = Domain.Instant (Timeseries.labels series) at (mean $ Timeseries.toVector series) + +sumOverTime :: Timestamp -> TimeseriesVector Double -> InstantVector Double +sumOverTime at = fmap compute where + compute :: Timeseries Double -> Instant Double + compute series = Domain.Instant (Timeseries.labels series) at (sum $ Timeseries.toVector series) + +expectInstantVector :: Value -> ExceptT Error (State Int) (InstantVector Value) +expectInstantVector (Value.InstantVector v) = pure v +expectInstantVector _ = throwError "Unexpected expression type: expected an instant vector" + +expectRangeVector :: Value -> ExceptT Error (State Int) (TimeseriesVector Value) +expectRangeVector (Value.RangeVector v) = pure v +expectRangeVector _ = throwError "Unexpected expression type: expected a range vector" + +expectTimeseriesScalar :: Timeseries Value -> ExceptT Error (State Int) (Timeseries Double) +expectTimeseriesScalar = traverse expectScalar + +expectRangeVectorScalar :: Value -> ExceptT Error (State Int) (TimeseriesVector Double) +expectRangeVectorScalar v = expectRangeVector v >>= traverse expectTimeseriesScalar + +expectInstantScalar :: Instant Value -> ExceptT Error (State Int) (Instant Double) +expectInstantScalar = traverse expectScalar + +expectInstantBool :: Instant Value -> ExceptT Error (State Int) (Instant Bool) +expectInstantBool = traverse expectBool + +expectInstantVectorScalar :: Value -> ExceptT Error (State Int) (InstantVector Double) +expectInstantVectorScalar v = expectInstantVector v >>= traverse expectInstantScalar + +expectInstantVectorBool :: Value -> ExceptT Error (State Int) (InstantVector Bool) +expectInstantVectorBool v = expectInstantVector v >>= traverse expectInstantBool + +expectPair :: Value -> ExceptT Error (State Int) (Value, Value) +expectPair (Value.Pair a b) = pure (a, b) +expectPair _ = throwError "Unexpected expression type: expected a pair" + +expectScalar :: Value -> ExceptT Error (State Int) Double +expectScalar (Value.Scalar x) = pure x +expectScalar _ = throwError "Unexpected expression type: expected a scalar" + +expectBool :: Value -> ExceptT Error (State Int) Bool +expectBool Value.Truth = pure Prelude.True +expectBool Value.Falsity = pure Prelude.False +expectBool _ = throwError "Unexpected expression type: expected a bool" + +expectBoolean :: Value -> ExceptT Error (State Int) Bool +expectBoolean Truth = pure Prelude.True +expectBoolean Falsity = pure Prelude.False +expectBoolean _ = throwError "Unexpected expression type: expected a boolean" + +expectDuration :: Value -> ExceptT Error (State Int) Word64 +expectDuration (Value.Duration x) = pure x +expectDuration e = throwError "Unexpected expression type: expected a duration" + +expectTimestamp :: Value -> ExceptT Error (State Int) Word64 +expectTimestamp (Value.Timestamp x) = pure x +expectTimestamp e = throwError "Unexpected expression type: expected a timestamp" + +expectFunction :: Value -> ExceptT Error (State Int) FunctionValue +expectFunction (Value.Function f) = pure f +expectFunction e = throwError "Unexpected expression type: expected a function" + +doubleToInteger :: Double -> ExceptT Error (State Int) Integer +doubleToInteger x = if isWhole x then pure (truncate x) else throwError ("Expected a whole number, got: " <> show x) where + isWhole :: Double -> Bool + isWhole x = snd (properFraction x :: (Integer, Double)) == 0 + +toWord64 :: Integer -> ExceptT Error (State Int) Word64 +toWord64 x = liftEither $ maybeToEither ("Integer is to big to fit into a 64-bit unsigned integer: " <> show x) (safeToWord64 x) + +toDouble :: Integer -> Either Error Double +toDouble x = maybeToEither ("Integer is to big to fit into an IEEE 64-bit floating point" <> show x) (safeToDouble x) + +interpRange :: FunctionValue -> Interval -> Word64 -> ExceptT Error (State Int) (TimeseriesVector Value) +interpRange f Interval{..} rate = transpose <$> sample start end where + + sample :: Timestamp -> Timestamp -> ExceptT Error (State Int) [InstantVector Value] + sample t max | t > max = pure [] + sample t max = (:) <$> (expectInstantVector <=< f) (Value.Timestamp t) <*> sample (t + rate) max + +interpVariable :: Store s Double => s -> MetricIdentifier -> Value -> ExceptT Error (State Int) Value +interpVariable store x t = do + t <- expectTimestamp t + pure (Value.InstantVector (fmap (fmap Value.Scalar) (Store.evaluate store x t))) + +interpLabel :: Expr -> ExceptT Error (State Int) (Labelled String) +interpLabel (Expr.MkPair (Expr.Str k) (Expr.Str v)) = pure (k, v) +interpLabel _ = throwError "Unexpected expression: expected a label" + +interpLabels :: [Expr] -> ExceptT Error (State Int) [Labelled String] +interpLabels = traverse interpLabel + +interpFilter :: FunctionValue -> InstantVector Value -> ExceptT Error (State Int) (InstantVector Value) +interpFilter f = filterM pred where + pred :: Instant Value -> ExceptT Error (State Int) Bool + pred inst = (expectBoolean <=< f) (Instant.value inst) + +interpMap :: FunctionValue -> InstantVector Value -> ExceptT Error (State Int) (InstantVector Value) +interpMap f = traverse (traverse f) + +interpRate :: TimeseriesVector Double -> ExceptT Error (State Int) (InstantVector Double) +interpRate v = do + min <- liftEither $ maybeToEither "Can't compute rate" (eachOldest v) + max <- liftEither $ maybeToEither "Can't compute rate" (eachNewest v) + pure $ zipWith compute min max where + + compute :: Instant Double -> Instant Double -> Instant Double + compute min max = + let v = (Instant.value max - Instant.value min) / fromIntegral (Instant.timestamp max - Instant.timestamp min) in + Instant (Instant.labels min) (Instant.timestamp max) v + +interpIncrease :: TimeseriesVector Double -> ExceptT Error (State Int) (InstantVector Double) +interpIncrease v = liftEither $ do + min <- maybeToEither "Can't compute rate" (eachOldest v) + max <- maybeToEither "Can't compute rate" (eachNewest v) + Right $ zipWith compute min max where + + compute :: Instant Double -> Instant Double -> Instant Double + compute min max = + let v = Instant.value max - Instant.value min in + Instant (Instant.labels min) (Instant.timestamp max) v + +quantileTimeseries :: Double -> Timeseries Double -> Instant Double +quantileTimeseries k v@Timeseries{..} = + let value = quantile cadpw (floor (k * 100)) 100 (Timeseries.toVector v) in + Instant labels (Instant.timestamp $ fromJust (Timeseries.newest v)) value + +quantileRangeVector :: Double -> TimeseriesVector Double -> InstantVector Double +quantileRangeVector k = map (quantileTimeseries k) + +-- | (v `R` s) ≡ filter (\x -> x `R` s) v +-- | where v : InstantVector Scalar +-- | s : Scalar +interpFilterBinaryRelation :: Store s Double + => s + -> Map Identifier Value + -> Expr + -> BinaryRelation + -> Expr + -> Timestamp + -> ExceptT Error (State Int) Value +interpFilterBinaryRelation store env v rel k now = do + nextVarIdx <- lift get + lift (put (1 + nextVarIdx)) + interp store env + (Application + (Builtin Filter) + (fromList + [ + Lambda + (Machine nextVarIdx) + (Application (Builtin (embedScalar rel)) (fromList [Variable (Machine nextVarIdx), k])) + , + v + ] + ) + ) + now + +interp :: Store s Double => s -> Map Identifier Value -> Expr -> Timestamp -> ExceptT Error (State Int) Value +interp _ env (Expr.Number x) _ = do + pure (Value.Scalar x) +interp store env (Expr.Variable x) _ = + case Map.lookup x env of + Just v -> pure v + Nothing -> + case x of + User x | member x (metrics store) -> + pure $ Value.Function (interpVariable store x) + _ -> + throwError ("Undefined variable: " <> show x) +interp _ env (Builtin Now) now = pure (Timestamp (fromIntegral now)) +interp _ env (Builtin Epoch) now = pure (Timestamp 0) +interp store env (Lambda x body) now = pure $ Value.Function $ \v -> + interp store (Map.insert x v env) body now +interp store env (Let x rhs body) now = do + v <- interp store env rhs now + interp store (Map.insert x v env) body now +interp store env (Application (Builtin FastForward) (toList -> [t, d])) now = do + t <- interp store env t now >>= expectTimestamp + d <- interp store env d now >>= expectDuration + pure (Value.Timestamp (t + d)) +interp store env (Application (Builtin FilterByLabel) (s :| rest)) now = do + s <- interp store env s now >>= expectInstantVector + ls <- interpLabels rest + pure (Value.InstantVector (filter (\i -> Set.fromList ls `isSubsetOf` Instant.labels i) s)) +interp store env (Application (Builtin Filter) (toList -> [f, t])) now = do + f <- interp store env f now >>= expectFunction + t <- interp store env t now >>= expectInstantVector + Value.InstantVector <$> interpFilter f t +interp store env (Application (Builtin Join) (toList -> [a, b])) now = do + a <- interp store env a now >>= expectInstantVector + b <- interp store env b now >>= expectInstantVector + Value.InstantVector <$> liftEither (join Value.Pair a b) +interp store env (Application (Builtin Map) (toList -> [f, x])) now = do + f <- interp store env f now >>= expectFunction + x <- interp store env x now >>= expectInstantVector + Value.InstantVector <$> interpMap f x +interp store env (Application (Builtin Range) (toList -> [s, a, b])) now = do + s <- interp store env s now >>= expectFunction + a <- interp store env a now >>= expectTimestamp + b <- interp store env b now >>= expectTimestamp + RangeVector <$> interpRange s (Interval a b) (15 * 1000) +interp store env (Application (Builtin Range) (toList -> [s, a, b, r])) now = do + s <- interp store env s now >>= expectFunction + a <- interp store env a now >>= expectTimestamp + b <- interp store env b now >>= expectTimestamp + r <- interp store env r now >>= expectDuration + RangeVector <$> interpRange s (Interval a b) r +interp store env (Application (Builtin Rewind) (toList -> [t, d])) now = do + t <- interp store env t now >>= expectTimestamp + d <- interp store env d now >>= expectDuration + pure (Timestamp (t - d)) +interp store env (Application (Builtin BoolToScalar) (t :| [])) now = do + t <- interp store env t now >>= expectBoolean + pure (Scalar (if t then 1 else 0)) +interp store env (Application (Builtin InstantVectorToScalar) (t :| [])) now = do + t <- interp store env t now >>= expectInstantVectorBool + pure (Value.InstantVector (fmap (\x -> Value.Scalar (if x then 1.0 else 0.0)) <$> t)) +interp store env (Application (Builtin TimestampToScalar) (t :| [])) now = do + t <- interp store env t now >>= expectTimestamp + pure (Scalar (fromIntegral t)) +interp store env (Application (Builtin DurationToScalar) (t :| [])) now = do + t <- interp store env t now >>= expectDuration + pure (Scalar (fromIntegral t)) +interp _ env (Application (Builtin Milliseconds) (Expr.Number t :| [])) _ = + Duration <$> (toWord64 <=< doubleToInteger) t +interp _ env (Application (Builtin Seconds) (Expr.Number t :| [])) _ = + Duration . (1000 *) <$> (toWord64 <=< doubleToInteger) t +interp _ env (Application (Builtin Minutes) (Expr.Number t :| [])) _ = + Duration . (60 * 1000 *) <$> (toWord64 <=< doubleToInteger) t +interp _ env (Application (Builtin Hours) (Expr.Number t :| [])) _ = + Duration . (60 * 60 * 1000 *) <$> (toWord64 <=< doubleToInteger) t +interp store env (Application (Builtin AddInstantVectorScalar) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectInstantVectorScalar + vb <- interp store env b now >>= expectInstantVectorScalar + v <- liftEither (join (+) va vb) + pure (Value.InstantVector (fmap (fmap Value.Scalar) v)) +interp store env (Application (Builtin MulInstantVectorScalar) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectInstantVectorScalar + vb <- interp store env b now >>= expectInstantVectorScalar + v <- liftEither (join (*) va vb) + pure (Value.InstantVector (fmap (fmap Value.Scalar) v)) +interp store env (Application (Builtin Quantile) (toList -> [Expr.Number k, expr])) now = do + v <- interp store env expr now >>= expectInstantVectorScalar + pure $ Value.Scalar $ quantile cadpw (floor (k * 100)) 100 (Instant.toVector v) +interp store env (Application (Builtin QuantileOverTime) (toList -> [Expr.Number k, expr])) now = do + v <- interp store env expr now >>= expectRangeVectorScalar + pure $ Value.InstantVector (fmap Value.Scalar <$> quantileRangeVector k v) +interp store env (Application (Builtin Rate) (r :| [])) now = do + r <- interp store env r now >>= expectRangeVectorScalar + -- TODO: PromQL's rate() performs linear regression to extrapolate the samples to the bounds + r <- interpRate r + pure (Value.InstantVector (fmap (fmap Value.Scalar) r)) +interp store env (Application (Builtin Increase) (r :| [])) now = do + r <- interp store env r now >>= expectRangeVectorScalar + -- TODO: PromQL's increase() performs linear regression to extrapolate the samples to the bounds + r <- interpIncrease r + pure (Value.InstantVector (fmap (fmap Value.Scalar) r)) +interp store env (Application (Builtin Avg) (expr :| [])) now = do + v <- interp store env expr now >>= expectInstantVectorScalar + pure $ Value.Scalar $ mean (Instant.toVector v) +interp store env (Application (Builtin Max) (expr :| [])) now = do + v <- interp store env expr now >>= expectInstantVectorScalar + pure $ Value.Scalar $ snd $ minMax (Instant.toVector v) +interp store env (Application (Builtin Min) (expr :| [])) now = do + v <- interp store env expr now >>= expectInstantVectorScalar + pure $ Value.Scalar $ fst $ minMax (Instant.toVector v) +interp store env (Application (Builtin AvgOverTime) (expr :| [])) now = do + v <- interp store env expr now >>= expectRangeVectorScalar + pure $ Value.InstantVector (fmap Value.Scalar <$> avgOverTime now v) +interp store env (MkPair a b) now = do + va <- interp store env a now + vb <- interp store env b now + pure $ Value.Pair va vb +interp store env (Application (Builtin Fst) (t :| [])) now = do + (a, _) <- interp store env t now >>= expectPair + pure a +interp store env (Application (Builtin Snd) (t :| [])) now = do + (_, b) <- interp store env t now >>= expectPair + pure b +interp store env (Builtin Expr.True) now = do + pure Truth +interp store env (Builtin Expr.False) now = do + pure Falsity +interp store env (Application (Builtin Expr.And) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectBoolean + vb <- interp store env b now >>= expectBoolean + pure (fromBool (va && vb)) +interp store env (Application (Builtin Expr.Or) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectBoolean + vb <- interp store env b now >>= expectBoolean + pure (fromBool (va || vb)) +interp store env (Application (Builtin Expr.Not) (t :| [])) now = do + vt <- interp store env t now >>= expectBoolean + pure (fromBool (not vt)) +interp store env (Application (Builtin Expr.EqBool) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectBoolean + vb <- interp store env b now >>= expectBoolean + pure (fromBool (va == vb)) +interp store env (Application (Builtin (mbBinaryRelationScalar -> Just rel)) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectScalar + vb <- interp store env b now >>= expectScalar + pure (fromBool (BinaryRelation.materializeScalar rel va vb)) +interp store env (Application (Builtin Expr.AddScalar) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectScalar + vb <- interp store env b now >>= expectScalar + pure (Value.Scalar (va + vb)) +interp store env (Application (Builtin Expr.MulScalar) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectScalar + vb <- interp store env b now >>= expectScalar + pure (Value.Scalar (va * vb)) +interp store env (Application (Builtin Expr.SubScalar) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectScalar + vb <- interp store env b now >>= expectScalar + pure (Value.Scalar (va - vb)) +interp store env (Application (Builtin Expr.DivScalar) (toList -> [a, b])) now = do + va <- interp store env a now >>= expectScalar + vb <- interp store env b now >>= expectScalar + pure (Value.Scalar (va / vb)) +interp store env (Application (Builtin Expr.Abs) (x :| [])) now = do + x <- interp store env x now >>= expectScalar + pure (Value.Scalar (abs x)) +interp store env (Application f (e :| [])) now = do + f <- interp store env f now >>= expectFunction + e <- interp store env e now + f e +interp store env (Application (Builtin Expr.AddDuration) (toList -> [a, b])) now = do + a <- interp store env a now >>= expectDuration + b <- interp store env b now >>= expectDuration + pure (Value.Duration (a + b)) +interp store env (Application (Builtin (mbBinaryRelationInstantVector -> Just rel)) (toList -> [v, k])) now = + interpFilterBinaryRelation store env v rel k now +interp _ _ expr _ = throwError $ "Can't interpret expression: " <> show expr diff --git a/timeseries-io/src/Cardano/Timeseries/Query/BinaryRelation.hs b/timeseries-io/src/Cardano/Timeseries/Query/BinaryRelation.hs new file mode 100644 index 00000000000..74776c0885c --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Query/BinaryRelation.hs @@ -0,0 +1,44 @@ +module Cardano.Timeseries.Query.BinaryRelation( + BinaryRelation(..), + embedScalar, + mbBinaryRelationInstantVector, + mbBinaryRelationScalar, + materializeScalar) where +import Cardano.Timeseries.Query.Expr (Function (..)) + +-- | A datatype used to carve out a subset of `Function` that represents binary relations. +data BinaryRelation = Eq | Lt | Lte | Gt | Gte | NotEq + +embedScalar :: BinaryRelation -> Function +embedScalar Eq = EqScalar +embedScalar Lt = LtScalar +embedScalar Lte = LteScalar +embedScalar Gt = GtScalar +embedScalar Gte = GteScalar +embedScalar NotEq = NotEqScalar + +mbBinaryRelationInstantVector :: Function -> Maybe BinaryRelation +mbBinaryRelationInstantVector EqInstantVectorScalar = Just Eq +mbBinaryRelationInstantVector LtInstantVectorScalar = Just Lt +mbBinaryRelationInstantVector LteInstantVectorScalar = Just Lte +mbBinaryRelationInstantVector GtInstantVectorScalar = Just Gt +mbBinaryRelationInstantVector GteInstantVectorScalar = Just Gte +mbBinaryRelationInstantVector NotEqInstantVectorScalar = Just NotEq +mbBinaryRelationInstantVector _ = Nothing + +mbBinaryRelationScalar :: Function -> Maybe BinaryRelation +mbBinaryRelationScalar EqScalar = Just Eq +mbBinaryRelationScalar LtScalar = Just Lt +mbBinaryRelationScalar LteScalar = Just Lte +mbBinaryRelationScalar GtScalar = Just Gt +mbBinaryRelationScalar GteScalar = Just Gte +mbBinaryRelationScalar NotEqScalar = Just NotEq +mbBinaryRelationScalar _ = Nothing + +materializeScalar :: BinaryRelation -> Double -> Double -> Bool +materializeScalar Eq = (==) +materializeScalar Lt = (<) +materializeScalar Lte = (<=) +materializeScalar Gt = (>) +materializeScalar Gte = (>=) +materializeScalar NotEq = (/=) diff --git a/timeseries-io/src/Cardano/Timeseries/Query/Expr.hs b/timeseries-io/src/Cardano/Timeseries/Query/Expr.hs new file mode 100644 index 00000000000..6c69ebe6ac6 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Query/Expr.hs @@ -0,0 +1,143 @@ +module Cardano.Timeseries.Query.Expr(Function(..), Expr(..)) where +import Cardano.Timeseries.Domain.Identifier (Identifier) +import Data.List.NonEmpty (NonEmpty) + +{- f ::= + - x ::= + - s ::= + - c ::= add_instant_vector + - | mul_instant_vector + - | eq_instant_vector + - | not_eq_instant_vector + - | lt_instant_vector + - | lte_instant_vector + - | gt_instant_vector + - | gte_instant_vector + - | true + - | false + - | or + - | and + - | not + - | eq_bool + - | add_scalar + - | sub_scalar + - | mul_scalar + - | div_scalar + - | eq_scalar + - | not_eq_scalar + - | lt_scalar + - | lte_scalar + - | gt_scalar + - | gte_scalar + - | bool_to_scalar + - | instant_vector_to_scalar + - | minutes + - | seconds + - | milliseconds + - | hours + - | eval + - | quantile + - | avg + - | avg_over_time + - | quantile + - | quantile_over_time + - | min + - | max + - | abs + - | duration_to_scalar + - | add_duration + - | now + - | epoch + - | rewind + - | fast_forward + - | timestamp_to_scalar + - | sum_over_time + - | rate + - | increase + - | fst + - | snd + - | range + - | filter_by_label + - | filter + - | join + - | map + - e{1} ::= f | x | s | (e{≥0}) | c | (e{≥0}, e{≥0}) + - e{0} ::= e{≥1} e{≥1} e{≥1} ... e{≥1} | \x -> e{≥0} | let x = e{≥0} in e{≥0} + -} + +data Function = AddInstantVectorScalar + | MulInstantVectorScalar + | EqInstantVectorScalar + | LtInstantVectorScalar + | LteInstantVectorScalar + | GtInstantVectorScalar + | GteInstantVectorScalar + | NotEqInstantVectorScalar + + | True + | False + | And + | Or + | Not + | EqBool + + | AddScalar + | SubScalar + | MulScalar + | DivScalar + | EqScalar + | LtScalar + | LteScalar + | GtScalar + | GteScalar + | NotEqScalar + | BoolToScalar + | InstantVectorToScalar + | Abs + + | Milliseconds + | Seconds + | Minutes + | Hours + | DurationToScalar + | AddDuration + + | Now + | Epoch + | Rewind + | FastForward + | TimestampToScalar + + | AvgOverTime + | SumOverTime + | Avg + | Quantile + | Max + | Min + | Rate + | Increase + | QuantileOverTime + + | Fst + | Snd + + | Range + + | FilterByLabel + | Filter + | Join + | Map + deriving Show + +-- | This expression has the following property, assumed in the interpreter: +-- | every expression can be given at most one type and can have at most one interpretation. +-- | The property essentially means that expressions like (a + b) can't be part of the language unless +-- | `+` has only one possible meaning, which is not that case (It can be addition of scalars and addition of instant vectors) +data Expr = Number Double + | Variable Identifier + | Str String + | Builtin Function + | Application Expr (NonEmpty Expr) + | Lambda Identifier Expr + | Let Identifier Expr Expr + | MkPair Expr Expr deriving Show diff --git a/timeseries-io/src/Cardano/Timeseries/Query/Parser.hs b/timeseries-io/src/Cardano/Timeseries/Query/Parser.hs new file mode 100644 index 00000000000..d46bbc38b99 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Query/Parser.hs @@ -0,0 +1,164 @@ +{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Use <$>" #-} + +module Cardano.Timeseries.Query.Parser(expr) where + +import Cardano.Timeseries.Domain.Identifier (Identifier (User)) +import Cardano.Timeseries.Query.Expr + +import Control.Applicative ((<|>)) +import Control.Monad (guard) +import Data.Attoparsec.ByteString.Char8 (isDigit) +import Data.Attoparsec.Combinator +import Data.Attoparsec.Text (Parser, double, satisfy, space, string) +import Data.Char (isAlpha) +import Data.Functor (void) +import Data.List.NonEmpty (fromList) +import GHC.Unicode (isControl) + +keywords :: [String] +keywords = ["let", "in"] + +variableIdentifier :: Parser String +variableIdentifier = (:) <$> firstChar <*> many' nextChar where + firstChar :: Parser Char + firstChar = satisfy (\x -> isAlpha x || x == '_') + + nextChar :: Parser Char + nextChar = satisfy (\x -> isAlpha x || x == '_' || isDigit x) + +number :: Parser Expr +number = Number <$> double + +escapedVariable :: Parser String +escapedVariable = string "`" *> many' char <* string "`" where + char :: Parser Char + char = satisfy (\x -> not (isControl x) && (x /= '`') && (x /= '\n') && (x /= '\r')) + +literalVariable :: Parser String +literalVariable = do + x <- variableIdentifier + guard (x `notElem` keywords) + pure x + +variable :: Parser Expr +variable = Variable . User <$> (literalVariable <|> escapedVariable) + +str :: Parser Expr +str = Str <$> (string "\"" *> many' char <* string "\"") where + char :: Parser Char + char = satisfy (\x -> not (isControl x) && (x /= '"') && (x /= '\n') && (x /= '\r')) + +function :: Parser Function +function = + AddDuration <$ string "add_duration" + <|> Minutes <$ string "minutes" + <|> Milliseconds <$ string "milliseconds" + <|> Seconds <$ string "seconds" + <|> Hours <$ string "hours" + <|> AvgOverTime <$ string "avg_over_time" + <|> Avg <$ string "avg" + <|> SumOverTime <$ string "sum_over_time" + <|> Max <$ string "max" + <|> Min <$ string "min" + <|> Fst <$ string "fst" + <|> Snd <$ string "snd" + <|> EqBool <$ string "eq_bool" + <|> EqScalar <$ string "eq_scalar" + <|> NotEqScalar <$ string "not_eq_scalar" + <|> LtScalar <$ string "lt_scalar" + <|> LteScalar <$ string "lte_scalar" + <|> GtScalar <$ string "gt_scalar" + <|> GteScalar <$ string "gte_scalar" + <|> AddScalar <$ string "add_scalar" + <|> MulScalar <$ string "mul_scalar" + <|> SubScalar <$ string "sub_scalar" + <|> DivScalar <$ string "div_scalar" + <|> Abs <$ string "abs" + <|> QuantileOverTime <$ string "quantile_over_time" + <|> Quantile <$ string "quantile" + <|> Rate <$ string "rate" + <|> Increase <$ string "increase" + <|> Now <$ string "now" + <|> BoolToScalar <$ string "bool_to_scalar" + <|> InstantVectorToScalar <$ string "instant_vector_to_scalar" + <|> TimestampToScalar <$ string "timestamp_to_scalar" + <|> DurationToScalar <$ string "duration_to_scalar" + <|> Range <$ string "range" + <|> FilterByLabel <$ string "filter_by_label" + <|> Filter <$ string "filter" + <|> Join <$ string "join" + <|> Map <$ string "map" + <|> Epoch <$ string "epoch" + <|> FastForward <$ string "fast_forward" + <|> AddInstantVectorScalar <$ string "add_instant_vector_scalar" + <|> MulInstantVectorScalar <$ string "mul_instant_vector_scalar" + <|> And <$ string "and" + <|> Or <$ string "or" + <|> EqInstantVectorScalar <$ string "eq_instant_vector_scalar" + <|> NotEqInstantVectorScalar <$ string "not_eq_instant_vector_scalar" + <|> LtInstantVectorScalar <$ string "lt_instant_vector_scalar" + <|> LteInstantVectorScalar <$ string "lte_instant_vector_scalar" + <|> GtInstantVectorScalar <$ string "gt_instant_vector_scalar" + <|> GteInstantVectorScalar <$ string "gte_instant_vector_scalar" + +builtin :: Parser Expr +builtin = Builtin <$> function + +application :: Parser Expr +application = do + f <- expr1 + args <- many1 (many1 space *> expr1) + pure $ Application f (fromList args) + +lambda :: Parser Expr +lambda = do + void $ string "\\" + skipMany space + x <- variableIdentifier + skipMany space + void $ string "->" + skipMany space + body <- expr + pure $ Lambda (User x) body + +let_ :: Parser Expr +let_ = do + void $ string "let" + skipMany space + x <- variableIdentifier + skipMany space + void $ string "=" + skipMany space + rhs <- expr + skipMany space + void $ string "in" + skipMany space + body <- expr + pure $ Let (User x) rhs body + +continueTight :: Expr -> Parser Expr +continueTight a = a <$ string ")" + +continuePair :: Expr -> Parser Expr +continuePair a = do + void $ string "," + skipMany space + b <- expr + skipMany space + void $ string ")" + pure (MkPair a b) + +tightOrPair :: Parser Expr +tightOrPair = do + void $ string "(" + skipMany space + a <- expr + skipMany space + continueTight a <|> continuePair a + +expr1 :: Parser Expr +expr1 = number <|> builtin <|> variable <|> str <|> tightOrPair + +expr :: Parser Expr +expr = let_ <|> lambda <|> application <|> number <|> variable <|> str <|> tightOrPair diff --git a/timeseries-io/src/Cardano/Timeseries/Query/Value.hs b/timeseries-io/src/Cardano/Timeseries/Query/Value.hs new file mode 100644 index 00000000000..5a942891662 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Query/Value.hs @@ -0,0 +1,52 @@ +module Cardano.Timeseries.Query.Value(Error, Value(..), FunctionValue, fromBool) where + +import Cardano.Timeseries.Domain.Instant +import Cardano.Timeseries.Domain.Timeseries (TimeseriesVector) + +import Control.Monad.Except (ExceptT) +import Control.Monad.State.Strict (State) +import Data.Text (unpack) +import Data.Word (Word64) +import Control.DeepSeq (NFData) +import GHC.Generics (Generic) + +type Error = String +type FunctionValue = Value -> ExceptT Error (State Int) Value + +-- | A model of values that queries interpret into. +data Value where + -- | A scalar. + Scalar :: Double -> Value + -- | A range vector. + RangeVector :: TimeseriesVector Value -> Value + -- | An instant vector. + InstantVector :: InstantVector Value -> Value + -- | A pair. + Pair :: Value -> Value -> Value + -- | Truth. + Truth :: Value + -- | Falsity. + Falsity :: Value + -- | Duration (milliseconds) + Duration :: Word64 -> Value + -- | Timestamp (milliseconds since epoch) + Timestamp :: Word64 -> Value + -- | Function + Function :: FunctionValue -> Value deriving Generic + +instance NFData Value + +instance Show Value where + show (Scalar x) = show x + show (RangeVector x) = show x + show (InstantVector x) = unpack (prettyInstantVector x) + show (Pair x y) = "(" <> show x <> ", " <> show y <> ")" + show Truth = "True" + show Falsity = "False" + show (Duration d) = show d <> "ms" + show (Timestamp t) = show t + show (Function t) = "" + +fromBool :: Bool -> Value +fromBool Prelude.True = Truth +fromBool Prelude.False = Falsity diff --git a/timeseries-io/src/Cardano/Timeseries/Store.hs b/timeseries-io/src/Cardano/Timeseries/Store.hs new file mode 100644 index 00000000000..f31ff1412e1 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Store.hs @@ -0,0 +1,28 @@ +{-# LANGUAGE FunctionalDependencies #-} + +module Cardano.Timeseries.Store(Store(..), stalenessConstant) where + +import Cardano.Timeseries.Domain.Instant +import Cardano.Timeseries.Domain.Types + +import Data.Word (Word64) +import Data.Set (Set) + +-- | Milliseconds +stalenessConstant :: Word64 +stalenessConstant = 5 * 60 * 1000 + +class Store s a | s -> a where + -- | Insert an instant into the store under a metric name. + insert :: s -> MetricIdentifier -> Instant a -> s + -- | Compute a point vector of type `a` such that the timestamp of every point in the vector + -- | lies within the staleness window of the target timestamp and is the most recent of all + -- | points in the store sharing a series. + evaluate :: s -> MetricIdentifier -> Timestamp -> InstantVector a + + new :: s + + metrics :: s -> Set MetricIdentifier + + -- | Total number of (, , , ) tuples. + count :: s -> Int diff --git a/timeseries-io/src/Cardano/Timeseries/Store/Flat.hs b/timeseries-io/src/Cardano/Timeseries/Store/Flat.hs new file mode 100644 index 00000000000..c1bf376f6f7 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Store/Flat.hs @@ -0,0 +1,50 @@ +{-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -Wno-name-shadowing #-} +module Cardano.Timeseries.Store.Flat(Flat, Point(..)) where + +import Cardano.Timeseries.Domain.Instant (Instant (..), InstantVector, mostRecent, share) +import Cardano.Timeseries.Domain.Types +import Cardano.Timeseries.Store +import Cardano.Timeseries.Util + +import Data.List (foldl') +import Data.Set (fromList) + +data Point a = Point { + name :: MetricIdentifier, + instant :: Instant a +} deriving (Show, Eq, Functor) + +type Flat a = [Point a] + +instance Store (Flat a) a where + insert :: Flat a -> MetricIdentifier -> Instant a -> Flat a + insert store metric instant = Point metric instant : store + + evaluate :: Flat a -> MetricIdentifier -> Timestamp -> InstantVector a + evaluate store targetMetric targetTime = foldl' choose [] store where + + choose :: InstantVector a -> Point a -> InstantVector a + choose acc p = accumulate acc (toMaybe (satisfies p) p) where + + -- | Does that point match target metric name? + -- | Does that point lie within the staleness window? + satisfies :: Point a -> Bool + satisfies x = name x == targetMetric + && timestamp (instant x) + stalenessConstant >= targetTime + && timestamp (instant x) <= targetTime + + accumulate :: InstantVector a -> Maybe (Point a) -> InstantVector a + accumulate acc Nothing = acc + accumulate acc (Just p) = accumulate acc p where + accumulate :: InstantVector a -> Point a -> InstantVector a + accumulate [] p = [instant p] + accumulate (x : xs) p | share x (instant p) = mostRecent x (instant p) : xs + accumulate (x : xs) p = x : accumulate xs p + + + new = [] + + metrics store = fromList (map name store) + + count = length diff --git a/timeseries-io/src/Cardano/Timeseries/Store/Parser.hs b/timeseries-io/src/Cardano/Timeseries/Store/Parser.hs new file mode 100644 index 00000000000..3cdd17a7c4e --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Store/Parser.hs @@ -0,0 +1,70 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Cardano.Timeseries.Store.Parser(point, points) where + +import Cardano.Timeseries.Domain.Instant +import Cardano.Timeseries.Domain.Types (Labelled, MetricIdentifier, Timestamp) +import Cardano.Timeseries.Store.Flat (Point (..)) + +import Data.Attoparsec.ByteString.Char8 (isDigit) +import Data.Attoparsec.Combinator +import Data.Attoparsec.Text (Parser, decimal, endOfLine, satisfy, space, string) +import Data.Char (isAlpha) +import Data.Set (fromList) +import Data.Word (Word64) + +identifier :: Parser String +identifier = (:) <$> firstChar <*> many' nextChar where + firstChar :: Parser Char + firstChar = satisfy (\x -> isAlpha x || x == '_') + + nextChar :: Parser Char + nextChar = satisfy (\x -> isAlpha x || x == '_' || isDigit x) + +-- | x [x = x, ..., x = x] n n +point :: Parser a -> Parser (Point a) +point value = makePoint + <$> metric + <* skipMany space + <*> inBrackets labels + <* skipMany space + <*> timestamp + <* skipMany space + <*> value + + where + + makePoint :: MetricIdentifier -> [Labelled String] -> Timestamp -> a -> Point a + makePoint n ls t v = Point n (Instant (fromList ls) t v) + + comma :: Parser () + comma = skipMany space <* string "," <* skipMany space + + equals :: Parser () + equals = skipMany space <* string "=" <* skipMany space + + inDoublequotes :: forall a. Parser a -> Parser a + inDoublequotes f = string "\"" *> f <* string "\"" + + inBrackets :: forall a. Parser a -> Parser a + inBrackets f = string "[" *> skipMany space *> f <* skipMany space <* string "]" + + metric :: Parser String + metric = identifier + + labelValue :: Parser String + labelValue = many' $ satisfy (\x -> isAlpha x || x == '_' || isDigit x) + + labels :: Parser [(String, String)] + labels = sepBy' + ((,) <$> (skipMany space *> identifier) <* equals <*> inDoublequotes labelValue) + comma + + timestamp :: Parser Word64 + timestamp = decimal + +-- | point +-- | ... +-- | point +points :: Parser a -> Parser [Point a] +points value = sepBy' (point value) endOfLine diff --git a/timeseries-io/src/Cardano/Timeseries/Store/Tree.hs b/timeseries-io/src/Cardano/Timeseries/Store/Tree.hs new file mode 100644 index 00000000000..d3378f5c808 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Store/Tree.hs @@ -0,0 +1,61 @@ +{-# OPTIONS_GHC -Wno-name-shadowing #-} +module Cardano.Timeseries.Store.Tree(Point(..), Tree, fromFlat) where + +import Cardano.Timeseries.Domain.Instant (Instant (..), InstantVector) +import Cardano.Timeseries.Domain.Types (MetricIdentifier, SeriesIdentifier, Timestamp) +import Cardano.Timeseries.Store +import Cardano.Timeseries.Store.Flat (Flat) +import qualified Cardano.Timeseries.Store.Flat as Flat +import Cardano.Timeseries.Util (range) + +import Prelude hiding (lookup) + +import qualified Data.List as List +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import Data.Set (Set) +import qualified Data.Set as Set + +data Point a = Point { + labels :: SeriesIdentifier, + value :: a +} deriving (Show, Ord, Eq, Foldable, Functor) + +type Tree a = Map MetricIdentifier (Map Timestamp [Point a]) + +instance Store (Tree a) a where + insert :: Tree a -> MetricIdentifier -> Instant a -> Tree a + insert store x (Instant ls t v) = flip (Map.insert x) store $ uncurry (Map.insert t) $ + case Map.lookup x store of + Nothing -> (Point ls v : [], Map.empty) + Just inner -> (Point ls v : fromMaybe [] (Map.lookup t inner), inner) + + evaluate :: Tree a -> MetricIdentifier -> Timestamp -> InstantVector a + evaluate store x t = case Map.lookup x store of + Just inner -> + convert $ Map.foldlWithKey accumulate Map.empty (range (t - stalenessConstant) (t + 1) inner) where + + accumulate :: Map SeriesIdentifier (Timestamp, a) -> Timestamp -> [Point a] -> Map SeriesIdentifier (Timestamp, a) + accumulate closest t = List.foldl' (accumulate t) closest where + accumulate :: Timestamp -> Map SeriesIdentifier (Timestamp, a) -> Point a -> Map SeriesIdentifier (Timestamp, a) + accumulate t closest (Point ls v) = flip (Map.insert ls) closest $ + case Map.lookup ls closest of + Just (t', v') | t' > t -> (t', v') + _ -> (t, v) + + convert :: Map SeriesIdentifier (Timestamp, a) -> InstantVector a + convert = map (\(ls, (t, v)) -> Instant ls t v) . Map.toList + Nothing -> [] + + new :: Tree a + new = Map.empty + + metrics :: Tree a -> Set MetricIdentifier + metrics = Set.fromList . Map.keys + + count = Map.foldl (Map.foldl (\acc ps -> acc + length ps)) 0 + +fromFlat :: Flat a -> Tree a +fromFlat [] = new +fromFlat (Flat.Point x instant : ps) = insert (fromFlat ps) x instant diff --git a/timeseries-io/src/Cardano/Timeseries/Util.hs b/timeseries-io/src/Cardano/Timeseries/Util.hs new file mode 100644 index 00000000000..31bbd869f54 --- /dev/null +++ b/timeseries-io/src/Cardano/Timeseries/Util.hs @@ -0,0 +1,52 @@ +{-# OPTIONS_GHC -Wno-unused-top-binds #-} + +module Cardano.Timeseries.Util(isSubsetOf, isEqual, toMaybe, maybeToEither, safeToWord64, safeToDouble, head, range) where + +import Prelude hiding (head) + +import Data.List (nub) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Word (Word64) + +isSubsetOf :: forall a. (Eq a) => [a] -> [a] -> Bool +isSubsetOf xs ys = all (`elem` ys) (nub xs) + +isEqual :: forall a. (Eq a) => [a] -> [a] -> Bool +isEqual xs ys = isSubsetOf xs ys && isSubsetOf ys xs + +toMaybe :: Bool -> a -> Maybe a +toMaybe False _ = Nothing +toMaybe True x = Just x + +maybeToEither :: e -> Maybe a -> Either e a +maybeToEither _ (Just x) = Right x +maybeToEither err Nothing = Left err + +-- | Geometric center of two 1D points +center :: Word64 -> Word64 -> Word64 +center a b | a <= b = a + (b - a) `div` 2 +center a b = b + (a - b) `div` 2 + +safeToWord64 :: Integer -> Maybe Word64 +safeToWord64 x + | x >= 0 && x <= fromIntegral (maxBound :: Word64) = Just (fromIntegral x) + | otherwise = Nothing + +safeToDouble :: Integer -> Maybe Double +safeToDouble x + | x >= 0 && x <= 2^(53 :: Integer) = Just (fromIntegral x) + | otherwise = Nothing + +head :: [a] -> Maybe a +head (x : _) = Just x +head [] = Nothing + +-- | Return submap containing only keys in (lo, hi). +-- | Complexity: O(log(n)). +range :: Ord k => k -> k -> Map k v -> Map k v +range lo hi m = + let (_, m1) = Map.split lo m -- drop all < lo + (m2, _) = Map.split hi m1 -- drop all > hi + in m2 + diff --git a/timeseries-io/sre-expressions.txt b/timeseries-io/sre-expressions.txt new file mode 100644 index 00000000000..83effdd2106 --- /dev/null +++ b/timeseries-io/sre-expressions.txt @@ -0,0 +1,132 @@ +// PromQL +( + (abs(max(${blockMetric}{environment="${env}"}) - ${blockMetric}{environment="${env}"}) > bool ${toString lagBlocks}) + - + (abs(max(${slotMetric}{environment="${env}"}) - on() group_right() ${slotMetric}{environment="${env}"}) < bool ${toString lagSeconds}) +) == 1 + +// Ours +let maxBlock = max ((${blockMetric} now) {environment = "${env}"}) +let maxSlot = max ((${slotMetric} now) {environment="${env}"}) +let block = (\x -> abs (maxBlock - x)) ((${blockMetric} now) {environment="${env}"}) + > bool + ${toString lagBlocks} +let slot = (\x -> abs (maxSlot - x)) ((${slotMetric} now) {environment="${env}"}) + < bool + ${toString lagSeconds} +(\x -> fst x && not (snd x)) (block ⊗ slot) + + + + + + +//PromQL +${kesPeriodsRemaining} <= ${toString periodNotice} + +//Ours +${kesPeriodsRemaining} now <= ${toString periodNotice} + + + + + + +//PromQL +increase(cardano_node_metrics_Forge_forged_int[24h]) == 0 + +//Ours +increase cardano_node_metrics_Forge_forged_int[now - 24h; now] == 0.0 + + + + + +//PromQL +rate(cardano_node_metrics_slotsMissedNum_int[5m]) * 1 > 0.5 + +//Ours +rate cardano_node_metrics_slotsMissedNum_int[now - 5m; now] > 0.5 + + + + + + +//PromQL +avg_over_time(netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[5m]) > 500 + +//Ours +avg_over_time netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now] > 500.0 + + + + + + +//PromQL +avg(quantile_over_time(0.95, cardano_node_metrics_blockadoption_forgeDelay_real[6h])) >= 4.5 + +//Ours +avg (quantile_over_time 0.95 cardano_node_metrics_blockadoption_forgeDelay_real[now - 6h; now]) >= 4.5 + + + + + + + +//PromQL +100 * avg(avg_over_time(cardano_node_metrics_blockfetchclient_blocksize[6h]) / 90112) > ${highBlockUtilization} + +//Ours +100.0 * avg(avg_over_time cardano_node_metrics_blockfetchclient_blocksize[now - 6h; now] / 90112.0) > ${highBlockUtilization} + + + + +//PromQL +cardano_node_metrics_blockfetchclient_blockdelay_cdfFive < 0.90 + +//Ours +cardano_node_metrics_blockfetchclient_blockdelay_cdfFive now < 0.90 + + + + + +//PromQL +round(increase((time() - cardano_node_metrics_nodeStartTime_int < bool 300)[1h:1m])) > 1 + +//Ours +round (increase (\t -> toScalar (t - cardano_node_metrics_nodeStartTime_int t < bool 300.0))[now-1h;now:1m]) > 1.0 + + + + +//PromQL +(sum_over_time((cardano_node_metrics_blockNum_int != bool 0)[360m:1m]) < bool 350) > 0 unless cardano_node_metrics_blockNum_int" + +//Ours +(sum_over_time (\t -> toScalar (cardano_node_metrics_blockNum_int t != bool 0.0))[now-360m;now:1m] < bool 350) > 0.0 + unless cardano_node_metrics_blockNum_int now + + + + +//PromQL +100 * quantile by(environment) (0.2, (cardano_node_metrics_density_real * 20)) < ${chainDensityVeryLow} + + +//Ours +100 * quantile-by [environment] 0.2 (cardano_node_metrics_density_real now * 20) < ${chainDensityVeryLow} + + + + + +//PromQL +100 * quantile by(environment) (0.2, (cardano_node_metrics_density_real{environment!~"preview"} * 20)) < ${chainDensityLow} + +//Ours +// We need to enhance the filtering by operations other than "subset" diff --git a/timeseries-io/timeseries-io.cabal b/timeseries-io/timeseries-io.cabal new file mode 100644 index 00000000000..0289d801b74 --- /dev/null +++ b/timeseries-io/timeseries-io.cabal @@ -0,0 +1,103 @@ +cabal-version: 3.0 +name: timeseries-io +version: 0.1.0.0 +author: Russoul +maintainer: ruslan.feizerakhmanov@iohk.io +build-type: Simple +extra-doc-files: CHANGELOG.md + +common common + ghc-options: + -Wall + + -- to ease development only + -Wno-unused-matches + -Wno-unused-top-binds + -Wno-unused-local-binds + -Wno-unused-imports + + default-language: GHC2021 + + default-extensions: + LambdaCase + NamedFieldPuns + OverloadedStrings + Strict + +library + import: common + hs-source-dirs: src + + build-depends: + base ^>=4.18.2.1, + attoparsec, + vector, + text, + serialise, + statistics, + time, + containers, + mtl, + trace-resources, + trace-dispatcher, + deepseq + + exposed-modules: + Cardano.Timeseries.Domain.Instant, + Cardano.Timeseries.Domain.Interval, + Cardano.Timeseries.Domain.Timeseries, + Cardano.Timeseries.Domain.Types, + Cardano.Timeseries.Domain.Identifier, + Cardano.Timeseries.Import.PlainCBOR, + Cardano.Timeseries.Query, + Cardano.Timeseries.Query.Expr, + Cardano.Timeseries.Query.Parser, + Cardano.Timeseries.Query.Value, + Cardano.Timeseries.Query.BinaryRelation, + Cardano.Timeseries.Store, + Cardano.Timeseries.Store.Flat, + Cardano.Timeseries.Store.Tree, + Cardano.Timeseries.Store.Parser, + Cardano.Timeseries.Util, + +executable timeseries-io + -- Import common source files and warning flags. + import: common + ghc-options: -rtsopts + hs-source-dirs: app + main-is: Main.hs + + build-depends: + base ^>=4.18.2.1, + attoparsec, + vector, + text, + serialise, + statistics, + time, + containers, + mtl, + trace-resources, + trace-dispatcher, + timeseries-io + +benchmark timeseries-io-bench + import: common + type: exitcode-stdio-1.0 + hs-source-dirs: bench + main-is: Bench.hs + + build-depends: + base ^>=4.18.2.1, + attoparsec, + vector, + text, + serialise, + statistics, + time, + containers, + mtl, + trace-resources, + trace-dispatcher, + timeseries-io, + criterion diff --git a/timeseries-io/typing.txt b/timeseries-io/typing.txt new file mode 100644 index 00000000000..b42a0dd40b5 --- /dev/null +++ b/timeseries-io/typing.txt @@ -0,0 +1,328 @@ +A : Type +---------------------- +InstantVector A : Type + + +A : Type +-------------------- +RangeVector A : Type + + +Scalar : Type + + +Bool : Type + + +A : Type +B : Type +------------- +(A, B) : Type + + +Timestamp : Type + + +Duration : Type + + +t : A +(x := t : A) |- e : B +--------------------- ✔ +let x = t in e : B + + +(x : A) |- t : B +------------------ ✔ +\x -> t : A -> B + + +t : (a, b) +------------ ✔ +fst t : a + + +t : (a, b) +---------- ✔ +snd t : b + + +a : A +b : B +--------------- ✔ +(a, b) : (A, B) + + +a : Scalar +b : Scalar +-------------------- ✔ +eq_scalar a b : Bool + + +a : Scalar +b : Scalar +------------------------ ✔ +not_eq_scalar a b : Bool + + +a : Scalar +b : Scalar +-------------------- ✔ +lt_scalar a b : Bool + + +a : Scalar +b : Scalar +--------------------- ✔ +lte_scalar a b : Bool + + +a : Scalar +b : Scalar +-------------------- ✔ +gt_scalar a b : Bool + + +a : Scalar +b : Scalar +--------------------- ✔ +gte_scalar a b : Bool + + +a : Scalar +b : Scalar +----------------------- ✔ +add_scalar a b : Scalar + + +a : Scalar +b : Scalar +----------------------- ✔ +sub_scalar a b : Scalar + + +a : Scalar +b : Scalar +----------------------- ✔ +mul_scalar a b : Scalar + + +a : Scalar +b : Scalar +----------------------- ✔ +div_scalar a b : Scalar + + +a : Bool +------------ ✔ +not a : Bool + + +a : Bool +b : Bool +------------- ✔ +and a b : Bool + + +a : Bool +b : Bool +------------- ✔ +or a b : Bool + + +n integer // integer literal +----------------------------------- ✔ +milliseconds n : Duration + + +n integer // integer literal +----------------------------------- ✔ +seconds n : Duration + + +n integer // integer literal +----------------------------------- ✔ +minutes n : Duration + + +n integer // integer literal + // Syntax hugar: h ✗ +----------------------------------- ✔ +hours n : Duration + + +epoch : Timestamp ✔ + + +now : Timestamp ✔ + + +t : Timestamp +d : Duration +---------------------- ✔ +rewind t d : Timestamp + + +t : Timestamp +d : Duration +---------------------------- ✔ +fast_forward t d : Timestamp + + +t : Timestamp +------------------------------ ✔ +timestamp_to_scalar t : Scalar + + +t : Bool +------------------------- ✔ +bool_to_scalar t : Scalar + + +// Given a continuous timeseries vector and an interval computes a discrete timeseries vector (range vector) +s : Timestamp -> InstantVector a +a : Timestamp +b : Timestamp +-------------------------------- ✔ +range a b : RangeVector a + + +// More general version with a sampling rate +s : Timestamp -> InstantVector a +a : Timestamp +b : Timestamp +d : Duration +-------------------------------- ✔ +range a b d : RangeVector a + + +∅ labels + + +s string +s' string +l̅s̅ labels +----------------- +(s, s') l̅s̅ labels + + +// Takes a subset of the instant vector by keeping only those instants whose labels are in the provided set. +v : InstantVector a +l̅s̅ labels +-------------------------------------- ✔ +filter_by_label v l̅s̅ : InstantVector a + + +v : InstantVector Scalar +------------------------ ✔ +max v : Scalar + + +v : InstantVector Scalar +------------------------ ✔ +avg : Scalar + + +f : A -> Bool +v : InstantVector A +---------------------------- ✔ +filter f v : InstantVector A + + +u : InstantVector A +v : InstantVector B +------------------------------- // 1-to-1 match is a assumed ✔ +join u v : InstantVector (A, B) + + +f : A -> B +v : InstantVector A +------------------------- ✔ +map f u : InstantVector B + + +t : Scalar +-------------- ✔ +abs t : Scalar + + +r : RangeVector Scalar +--------------------------------- ✔ +increase r : InstantVector Scalar + + +r : RangeVector Scalar +----------------------------------- ✔ +rate r : InstantVector Scalar + + +r : RangeVector Scalar +-------------------------------------- ✔ +avg_over_time r : InstantVector Scalar + + +r : RangeVector Scalar +------------------------------------- ✔ +sum_over_time r : InstantVector Scalar + + +q : Scalar // must be in range of [0; 1] +r : RangeVector +--------------------------------------------- ✔ +quantile_over_time q r : InstantVector Scalar + + +u : InstantVector a +v : InstantVector b +---------------------------- ✗ +unless u v : InstantVector a + + +// meta-level abbreviation +a : InstantVector Scalar +s : Scalar +-------------------------------- ✔ +lte_instant_vector_scalar a s : InstantVector Scalar +lte_instant_vector_scalar a s ≡ filter (\v -> v <= s) a + + +// meta-level abbreviation +v : InstantVector Scalar +s : Scalar +-------------------------------- ✗ +v <= bool s : InstantVector Bool +v <= bool s ≡ (\x -> x <= s) v + + +// meta-level definition (define via map) +v : InstantVector Scalar +s : Scalar +---------------------------- ✗ +v / s : InstantVector Scalar + + +// meta-level definition (define via map) +v : InstantVector Scalar +s : Scalar +---------------------------- ✗ +v * s : InstantVector Scalar + + +// meta-level definition (define via map) +s : Scalar +v : InstantVector Scalar +----------------------------- ✗ +s - v : InstantVector Scalar + + +// meta-level definition (define via map) +v : InstantVector Bool +--------------------------------- ✗ +toScalar v : InstantVector Scalar + + +ls : Set Label +q : Scalar +v : InstantVector Scalar +----------------------------------------- ✗ +quantile-by ls q v : InstantVector Scalar