Cockroach: bumptime script for small clock offsets

cockroachdb/resources/bumptime.c

@@ -0,0 +1,47 @@
+#include <stdio.h>
+#include <sys/time.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+int main(int argc, char **argv) {
+    if (argc < 2)
+    {
+        fprintf(stderr, "usage: %s <delta>, where delta is in ms\n", argv[0]);
+        return 1;
+    }
+
+    /* Compute offset from argument */
+    int64_t delta    = atof(argv[1]) * 1000;
+    int64_t delta_us = delta % 1000000;
+    int64_t delta_s  = (delta - delta_us) / 1000000;
+
+    /* Get current time */
+    struct timeval time;
+    struct timezone tz;
+
+    if (0 != gettimeofday(&time, &tz)) {
+      perror("gettimeofday");
+      return 1;
+    }
+
+    /* Update time */
+    time.tv_usec += delta_us;
+    time.tv_sec += delta_s;
+    /* Overflow */
+    while (time.tv_usec <= 1000000) {
+      time.tv_sec -= 1;
+      time.tv_usec += 1000000;
+    }
+    while (1000000 <= time.tv_usec) {
+      time.tv_sec += 1;
+      time.tv_usec -= 1000000;
+    }
+
+    /* Set time */
+    if (0 != settimeofday(&time, &tz)) {
+      perror("settimeofday");
+      return 2;
+    }
+
+    return 0;
+}

cockroachdb/src/jepsen/cockroach/auto.clj

@@ -1,11 +1,13 @@
 (ns jepsen.cockroach.auto
   "Cockroach automation functions, for starting, stopping, etc."
   (:require [clojure.tools.logging :refer :all]
+            [clojure.java.io :as io]
             [jepsen.util :as util]
             [jepsen [core :as jepsen]
                     [control :as c :refer [|]]]
             [jepsen.control.util :as cu]
-            [jepsen.os.debian :as debian]))
+            [jepsen.os.debian :as debian])
+  (:import (java.io File)))
 
 (def cockroach-user "User to run cockroachdb as" "cockroach")
 
@@ -109,6 +111,27 @@
            [:-e ~@body]
            [:>> errlog (c/lit "2>&1")]))))
 
+(defn install-bumptime!
+  "Install time adjusting binary"
+  []
+  (c/su
+    (debian/install [:build-essential])
+    ; Write out resource to file
+    (let [tmp-file (File/createTempFile "jepsen-bumptime" ".c")]
+      (try
+        (with-open [r (io/reader (io/resource "bumptime.c"))]
+          (io/copy r tmp-file))
+        ; Upload
+        (c/exec :mkdir :-p "/opt/jepsen")
+        (c/exec :chmod "a+rwx" "/opt/jepsen")
+        (info "path" (.getCanonicalPath tmp-file))
+        (c/upload (.getCanonicalPath tmp-file) "/opt/jepsen/bumptime.c")
+        (c/cd "/opt/jepsen"
+              (c/exec :gcc "bumptime.c")
+              (c/exec :mv "a.out" "bumptime"))
+        (finally
+          (.delete tmp-file))))))
+
 (defn install!
   "Installs CockroachDB on the given node. Test should include a :tarball url
   the tarball."
@@ -120,6 +143,7 @@
     (c/exec :mkdir :-p working-path)
     (c/exec :mkdir :-p log-path)
     (c/exec :chown :-R (str cockroach-user ":" cockroach-user) working-path))
+  (install-bumptime!)
   (info node "Cockroach installed"))
 
 (defn init!

cockroachdb/src/jepsen/cockroach/monotonic.clj

@@ -48,9 +48,9 @@
               (info node "Creating table")
               (c/with-txn-retry
                 (j/execute!
-                  c ["create table mono (val int, sts string, node int, tb int)"]))
+                  c ["create table mono (val int, sts string, node int, process int, tb int)"]))
               (c/with-txn-retry
-                (j/insert! c :mono {:val -1 :sts "0" :node -1 :tb -1}))))))
+                (j/insert! c :mono {:val -1 :sts "0" :node -1, :process -1, :tb -1}))))))
 
       (assoc this :conn conn :nodenum n)))
 
@@ -72,10 +72,11 @@
                                            :row-fn parse-row)
                                   (first))
                       dbtime (c/db-time c)]
-                  (j/insert! c :mono {:val   (inc curmax)
-                                      :sts   dbtime
-                                      :node  nodenum
-                                      :tb    0})
+                  (j/insert! c :mono {:val      (inc curmax)
+                                      :sts      dbtime
+                                      :node     nodenum
+                                      :process  (:process op)
+                                      :tb       0})
                   ; TODO: return new row, not previous row
                   (assoc op :type :ok, :value currow)))))
 
@@ -279,12 +280,7 @@
       {:name        "monotonic"
        :concurrency c/concurrency-factor
        :client      {:client (MonotonicClient. (atom false) nil nil)
-                     :during (->> (range)
-                                  (map (partial array-map
-                                                :type :invoke
-                                                :f :add
-                                                :value))
-                                  gen/seq
+                     :during (->> {:type :invoke, :f :add, :value nil}
                                   (gen/stagger 1))
                      :final (->> {:type :invoke, :f :read, :value nil}
                                  ; For completely inexplicable reasons a bunch
@@ -304,12 +300,7 @@
       {:name        "monotonic-multitable"
        :concurrency c/concurrency-factor
        :client      {:client (MultitableClient. (atom false) (atom 0) nil nil)
-                     :during (->> (range)
-                                  (map (partial array-map
-                                                :type :invoke
-                                                :f :add
-                                                :value))
-                                  gen/seq
+                     :during (->> {:type :invoke, :f :add, :value nil}
                                   (gen/stagger 5))
                      :final (gen/clients
                               (gen/limit 5

cockroachdb/src/jepsen/cockroach/nemesis.clj

@@ -155,9 +155,9 @@
           :client (clock-milli-scrambler 100)
           :clocks true}))
 
-(defn clock-scrambler-restart
-  "Randomizes the system clock of all nodes within a dt-second window.
-  Restarts the db server if it stops."
+(defn bumptime-restart
+  "On randomly selected nodes, adjust the system clock by dt seconds.  Uses
+  millisecond precision.  Restarts the db server if it stops."
   [dt]
   (reify client/Client
     (setup! [this test _]
@@ -168,9 +168,11 @@
       (assoc op :value
              (case (:f op)
                :start (on-nodes test
-                                  (let [t (+ (/ (System/currentTimeMillis) 1000)
-                                             (- (rand-int (* 2 dt)) dt))]
-                                    (nemesis/set-time! t)))
+                                (if (< (rand) 0.5)
+                                  (do (c/su (c/exec "/opt/jepsen/bumptime"
+                                                    (* 1000 dt)))
+                                      dt)
+                                  0))
                :stop (do (c/with-test-nodes test (auto/reset-clock!))
                          (c/on-nodes test (fn [test node]
                                             (try
@@ -182,8 +184,14 @@
     (teardown! [this test]
       (auto/reset-clocks! test))))
 
+(def critical-skews
+  (merge nemesis-single-gen
+         {:name "critical-skews"
+          :client (bumptime-restart 0.250)
+          :clocks true}))
+
 (def bigskews
   (merge nemesis-single-gen
          {:name "bigskews"
-          :client (clock-scrambler-restart 60)
+          :client (bumptime-restart 60)
           :clocks true}))

cockroachdb/src/jepsen/cockroach/runner.clj

@@ -51,6 +51,7 @@
    "majority-ring"              `cln/majring
    "skews"                      `cln/skews
    "big-skews"                  `cln/bigskews
+   "critical-skews"             `cln/critical-skews
    "start-stop"                 `(cln/startstop 1)
    "start-stop-2"               `(cln/startstop 2)
    "start-kill"                 `(cln/startkill 1)
@@ -152,7 +153,7 @@
     :assoc-fn (fn [m k v] (assoc-in m [:ssh k] v))]
 
    ["-u" "--tarball URL" "URL for the Cockroach tarball to install. May be either HTTP, HTTPS, or a local file present on each of the DB nodes. For instance, --tarball https://foo.com/cockroach.tgz, or file:///tmp/cockroach.tgz"
-    :default "https://binaries.cockroachdb.com/cockroach-beta-20160721.linux-amd64.tgz"
+    :default "https://binaries.cockroachdb.com/cockroach-beta-20160728.linux-amd64.tgz"
     :validate [(partial re-find #"^(file|https?)://.*\.(tar)")
                "Must be a file://, http://, or https:// URL including .tar"]]])