Merge pull request #292 from mkcp/master

Add clock nemesis to dgraph

dgraph/src/jepsen/dgraph/client.clj

@@ -26,7 +26,7 @@
 (def default-port "Default dgraph alpha GRPC port" 9080)
 
 ;; milliseconds given to the grpc blockingstub as a deadline
-(def deadline 20000)
+(def deadline 30000)
 
 (defn open
   "Creates a new DgraphClient for the given node."

dgraph/src/jepsen/dgraph/core.clj

@@ -42,7 +42,14 @@
     :fix-alpha?
     :partition-halves?
     :partition-ring?
-    :move-tablet?})
+    :move-tablet?
+    :skew-clock?})
+
+(def skew-specs
+  #{:tiny
+    :small
+    :big
+    :huge})
 
 (defn dgraph-test
   "Builds up a dgraph test map from CLI options."
@@ -145,6 +152,11 @@
                (str "Should be a comma-separated list of failure types. A failure type "
                     (.toLowerCase (cli/one-of nemesis-specs))
                     ". Or, you can use 'none' to indicate no failures.")]]
+   [nil "--skew SPEC" "Set the duration of clock skews"
+    :parse-fn keyword
+    :default :small
+    :assoc-fn (fn [m k v] (update m :nemesis assoc :skew v))
+    :validate [skew-specs (.toLowerCase (cli/one-of skew-specs))]]
    ["-f" "--force-download" "Ignore the package cache; download again."
     :default false]
    [nil "--upsert-schema"

dgraph/src/jepsen/dgraph/nemesis.clj

@@ -4,8 +4,10 @@
             [clojure.tools.logging :refer [info warn]]
             [jepsen [control :as c]
                     [generator :as gen]
+                    [net :as net]
                     [util :as util]
                     [nemesis :as nemesis]]
+            [jepsen.nemesis.time :as nt]
             [jepsen.control.util :as cu]
             [jepsen.dgraph [support :as s]]))
 
@@ -75,6 +77,38 @@
 
     (teardown! [this test])))
 
+(defn bump-time
+  "On randomly selected nodes, adjust the system clock by dt seconds.  Uses
+  millisecond precision."
+  [dt]
+  (reify nemesis/Nemesis
+    (setup! [this test]
+      (nt/reset-time! test)
+      this)
+
+    (invoke! [this test op]
+      (assoc op :value
+             (case (:f op)
+               :start (c/with-test-nodes test
+                        (if (< (rand) 0.5)
+                          (do (nt/bump-time! dt)
+                              dt)
+                          0))
+               :stop (c/with-test-nodes test
+                       (nt/reset-time!)))))
+
+    (teardown! [this test]
+      (nt/reset-time! test))))
+
+(defn skew
+  [{:keys [skew] :as opts}]
+  (case skew
+    :huge  (bump-time 7500)
+    :big   (bump-time 2000)
+    :small (bump-time 250)
+    :tiny  (bump-time 100)
+    (bump-time 0)))
+
 (defn full-nemesis
   "Can kill and restart all processes and initiate network partitions."
   [opts]
@@ -88,7 +122,9 @@
      {:start-partition-halves  :start
       :stop-partition-halves   :stop} (nemesis/partition-random-halves)
      {:start-partition-ring    :start
-      :stop-partition-ring     :stop} (nemesis/partition-majorities-ring)}))
+      :stop-partition-ring     :stop} (nemesis/partition-majorities-ring)
+     {:start-skew :start
+      :stop-skew  :stop} (skew opts)}))
 
 (defn op
   "Construct a nemesis op"
@@ -113,6 +149,8 @@
         (when (:partition-ring? opts)
           [(gen/seq (cycle (map op [:start-partition-ring
                                     :stop-partition-ring])))])
+        (when (:skew-clock? opts)
+          [(gen/seq (cycle (map op [:start-skew :stop-skew])))])
         (when (:move-tablet? opts)
           [(op :move-tablet)])]
        (apply concat)
@@ -126,6 +164,7 @@
    :generator (full-generator opts)
    :final-generator (->> [(when (:partition-halves opts) :stop-partition-halves)
                           (when (:partition-ring opts)   :stop-partition-ring)
+                          (when (:skew-clock? opts)      :stop-skew)
                           (when (:kill-zero?  opts)      :restart-zero)
                           (when (:kill-alpha? opts)      :restart-alpha)]
                          (remove nil?)

dgraph/src/jepsen/dgraph/support.clj

@@ -10,9 +10,13 @@
                     [control  :as c]
                     [core     :as jepsen]
                     [util     :refer [meh]]]
+            [jepsen.nemesis.time :as nt]
             [jepsen.control.util :as cu]
-            [jepsen.dgraph.client :as dc])
-  (:import (java.util.concurrent CyclicBarrier)))
+            [jepsen.os.debian :as debian]
+            [jepsen.dgraph.client :as dc]
+            [clojure.java.io :as io])
+  (:import (java.util.concurrent CyclicBarrier)
+           (java.io File)))
 
 ; Local paths
 (def dir "/opt/dgraph")
@@ -229,6 +233,8 @@
               dir
               (:force-download test)))
 
+          (nt/install!)
+
           (when (= node (jepsen/primary test))
             (start-zero! test node)
             ; TODO: figure out how long to block here