charybdefs: New subproject for disk fault injection

Currently builds and installs everything and provides some basic
controls for injecting faults. Future work includes building an
actual nemesis with this and providing more fine-grained controls.

charybdefs/.gitignore

@@ -0,0 +1,11 @@
+/target
+/classes
+/checkouts
+pom.xml
+pom.xml.asc
+*.jar
+*.class
+/.lein-*
+/.nrepl-port
+.hgignore
+.hg/

charybdefs/README.md

@@ -0,0 +1,13 @@
+# charybdefs
+
+A wrapper around [CharybdeFS](https://github.com/scylladb/charybdefs) for use
+in a jepsen nemesis.
+
+## Usage
+
+TODO
+
+## License
+
+Distributed under the Eclipse Public License either version 1.0 or (at
+your option) any later version.

charybdefs/project.clj

@@ -0,0 +1,8 @@
+(defproject jepsen-charybdefs "0.1.0-SNAPSHOT"
+  :description "charybdefs wrapper for use in jepsen"
+  :url "https://github.com/jepsen.io/jepsen"
+  :license {:name "Eclipse Public License"
+            :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :dependencies [[org.clojure/clojure "1.8.0"]
+                 [jepsen "0.1.6"]
+                 [yogthos/config "0.8"]])

charybdefs/src/jepsen/charybdefs.clj

@@ -0,0 +1,85 @@
+(ns jepsen.charybdefs
+  (:require [clojure.tools.logging :refer [info]]
+            [jepsen.control :as c]
+            [jepsen.control.util :as cu]
+            [jepsen.os.debian :as debian]))
+
+(defn install-thrift!
+  "Install thrift (compiler, c++, and python libraries) from source.
+
+  Ubuntu includes thrift-compiler and python-thrift packages, but
+  not the c++ library, so we must build it from source. We can't mix
+  and match versions, so we must install everything from source."
+  []
+  (when-not (cu/exists? "/usr/bin/thrift")
+    (c/su
+     (debian/install [:automake
+                      :bison
+                      :flex
+                      :g++
+                      :git
+                      :libboost-all-dev
+                      :libevent-dev
+                      :libssl-dev
+                      :libtool
+                      :make
+                      :pkg-config
+                      :python-setuptools
+                      :libglib2.0-dev])
+     (info "Building thrift (this takes several minutes)")
+     (let [thrift-dir "/opt/thrift"]
+       (cu/install-archive! "http://www-eu.apache.org/dist/thrift/0.10.0/thrift-0.10.0.tar.gz" thrift-dir)
+       (c/cd thrift-dir
+             ;; charybdefs needs this in /usr/bin
+             (c/exec "./configure" "--prefix=/usr")
+             (c/exec :make :-j4)
+             (c/exec :make :install))
+       (c/cd (str thrift-dir "/lib/py")
+             (c/exec :python "setup.py" :install))))))
+
+(defn install!
+  "Ensure CharybdeFS is installed and the filesystem mounted at /faulty."
+  []
+  (install-thrift!)
+  (let [charybdefs-dir "/opt/charybdefs"
+        charybdefs-bin (str charybdefs-dir "/charybdefs")]
+    (when-not (cu/exists? charybdefs-bin)
+      (c/su
+       (debian/install [:build-essential
+                        :cmake
+                        :libfuse-dev
+                        :fuse]))
+      (c/su
+       (c/exec :mkdir :-p charybdefs-dir)
+       (c/exec :chmod "777" charybdefs-dir))
+      (c/exec :git :clone :--depth 1 "https://github.com/scylladb/charybdefs.git" charybdefs-dir)
+      (c/cd charybdefs-dir
+            (c/exec :thrift :-r :--gen :cpp :server.thrift)
+            (c/exec :cmake :CMakeLists.txt)
+            (c/exec :make)))
+    (c/su
+     (c/exec :modprobe :fuse)
+     (c/exec :umount "/faulty" "||" "/bin/true")
+     (c/exec :mkdir :-p "/real" "/faulty")
+     (c/exec charybdefs-bin "/faulty" "-oallow_other,modules=subdir,subdir=/real")
+     (c/exec :chmod "777" "/real" "/faulty"))))
+
+(defn- cookbook-command
+  [flag]
+  (c/cd "/opt/charybdefs/cookbook"
+        (c/exec "./recipes" flag)))
+
+(defn break-all
+  "All operations fail with EIO."
+  []
+  (cookbook-command "--io-error"))
+
+(defn break-one-percent
+  "1% of disk operations fail."
+  []
+  (cookbook-command "--probability"))
+
+(defn clear
+  "Clear a previous failure injection."
+  []
+  (cookbook-command "--clear"))

charybdefs/test/.gitignore

@@ -0,0 +1 @@
+config.edn

charybdefs/test/jepsen/charybdefs/remote_test.clj

@@ -0,0 +1,29 @@
+(ns jepsen.charybdefs.remote-test
+  (:require [clojure.test :refer :all]
+            [jepsen.control :as c]
+            [jepsen.charybdefs :as charybdefs]
+            [config.core :refer [env]]))
+
+;;; To run these tests, create a file config.edn in ../.. (the 'test' directory)
+;;; containing at least:
+;;;   {:hostname "foo"}
+;;; This must name a host you can ssh to without a password, and have passwordless
+;;; sudo on. It must run debian or ubuntu (tested with ubuntu 16.04).
+;;; Other ssh options may also be used.
+(use-fixtures :each
+  (fn [f]
+    (if (not (:hostname env))
+      (throw (RuntimeException. "hostname is required in config.edn")))
+    (c/with-ssh (select-keys env [:private-key-path :strict-host-key-checking :username])
+      (c/on (:hostname env)
+            (charybdefs/install!)
+            (f)))))
+
+(deftest break-fix
+  (testing "break the disk and then fix it"
+    (let [filename "/faulty/foo"]
+      (c/exec :touch filename)
+      (charybdefs/break-all)
+      (is (thrown? RuntimeException (c/exec :cat filename)))
+      (charybdefs/clear)
+      (is "" (c/exec :cat filename)))))

jepsen/src/jepsen/nemesis.clj

@@ -298,28 +298,3 @@
       op)
 
     (teardown! [this test])))
-
-
-; Notes for filesystem fault injection... someday
-
-;(defn install-thrift!
-;  [test]
-; apt-get install automake bison flex g++ git libboost1.55-all-dev libevent-dev libssl-dev libtool make pkg-config libglib2.0-dev
-; cd /tmp
-; wget http://www-eu.apache.org/dist/thrift/0.10.0/thrift-0.10.0.tar.gz
-; tar xvfz thrift-0.10.0.tar.gz
-; cd thrift-0.10.0
-; ./configure
-; make
-  ; make install
-
-
-; apt-get install python-thrift build-essential cmake libfuse-dev fuse
-; cd /opt
-; git clone https://github.com/scylladb/charybdefs.git
-; cd charybdefs
-; thrift -r --gen cpp server.thrift
-; Their cmake script can't handle /usr/local/bin
-; ln -s /usr/local/bin/thrift /usr/bin/thrift
-; cmake CMakeLists.txt
-; make