Changes needed to compile and run code

Makefile

@@ -1,15 +1,15 @@
-CC=g++
+CC=g++ -faligned-new
 CFLAGS=-Wall -g -gdwarf-3 -std=c++0x 
 #CFLAGS += -fsanitize=address -fno-omit-frame-pointer 
 JEMALLOC=./jemalloc-4.0.3
-NNMSG=./nanomsg-0.5-beta
+NNMSG=./nanomsg-0.6-beta
 
 .SUFFIXES: .o .cpp .h
 
 SRC_DIRS = ./ ./benchmarks/ ./client/ ./concurrency_control/ ./storage/ ./transport/ ./system/ ./statistics/#./unit_tests/
-DEPS = -I. -I./benchmarks -I./client/ -I./concurrency_control -I./storage -I./transport -I./system -I./statistics -I$(JEMALLOC)/include #-I./unit_tests 
+DEPS = -I. -I./boost_1_79_0 -I./benchmarks -I./client/ -I./concurrency_control -I./storage -I./transport -I./system -I./statistics -I$(JEMALLOC)/include #-I./unit_tests 
 
-CFLAGS += $(DEPS) -D NOGRAPHITE=1 -Werror -Wno-sizeof-pointer-memaccess
+CFLAGS += $(DEPS) -D NOGRAPHITE=1 -Wno-sizeof-pointer-memaccess
 LDFLAGS = -Wall -L. -L$(NNMSG) -L$(JEMALLOC)/lib -Wl,-rpath,$(JEMALLOC)/lib -pthread -gdwarf-3 -lrt -std=c++0x
 #LDFLAGS = -Wall -L. -L$(NNMSG) -L$(JEMALLOC)/lib -Wl,-rpath,$(JEMALLOC)/lib -pthread -gdwarf-3 -lrt -std=c++11
 LDFLAGS += $(CFLAGS)

README.md

@@ -9,9 +9,20 @@ This testbed is based on the DBx1000 system, whose concurrency control scalabili
     Xiangyao Yu, George Bezerra, Andrew Pavlo, Srinivas Devadas, Michael Stonebraker
     http://voltdb.com/downloads/datasheets_collateral/vdb_whitepaper_staring_into_the_abyss.pdf
 
-Build & Test
+Setup
 ------------
+DDBMS has three dependences that need to be installed:
+
+* [Boost](https://www.boost.org/)
+* [jemalloc](https://github.com/jemalloc/jemalloc/releases/tag/4.0.3)
+* [nanomsg](https://github.com/nanomsg/nanomsg/releases/tag/0.6-beta)
+
+To be able to make the code successfully there needs to be a file named obj. Run 
+
+    mkdir obj
 
+Build & Test
+------------
 To build the database.
 
     make deps
@@ -43,3 +54,20 @@ The DBMS can be run with
     ./runcl -nid[M]
 
 where N and M are the ID of a server and client, respectively
+
+For example, if you are running with a `THREAD_CNT` of 4 you would run 
+
+    ./rundb -nid0
+    ./rundb -nid1
+    ./runcl -nid2
+    ./runcl -nid3
+
+There is also the option to run scripts. From the scripts directory run 
+
+    python3 run_experiments -e [experiment]
+
+* List of available experiments can be found [here](https://github.com/mitdbg/deneva/blob/master/scripts/experiments.py)
+
+After running an experiment, the results can be plotted by running 
+
+    python3 plot.py [experiment]

concurrency_control/row_lock.cpp

@@ -206,10 +206,11 @@ RC Row_lock::lock_get(lock_t type, TxnManager * txn, uint64_t* &txnids, int &txn
 INC_STATS(txn->get_thd_id(),twopl_getlock_time,timespan);
 INC_STATS(txn->get_thd_id(),twopl_getlock_cnt,1);
 
-    if (g_central_man)
+    if (g_central_man){
         glob_manager.release_row(_row);
-    else
-        pthread_mutex_unlock( latch );
+    }else{
+        pthread_mutex_unlock( latch );	
+    }
 
 
 	return rc;

scripts/draw.py

@@ -384,7 +384,7 @@ def draw_line(fname, data, xticks,
     if logscale :
         ax.set_yscale('log')
     if logscalex:
-        ax.set_xscale('log',basex=base)
+        ax.set_xscale('log')
     n = 0
     if xlabels != None :
         ax.set_xticklabels([x if i%2 else '' for x,i in zip(xlabels,range(len(xlabels)))]) 
@@ -432,17 +432,17 @@ def draw_line(fname, data, xticks,
             ax.get_xaxis().set_major_formatter(matplotlib.ticker.FormatStrFormatter('%d'))
     if legend :
         #fig.legend(lines, linenames, loc='upper right',bbox_to_anchor = (1,1), prop={'size':9}, ncol=ncol)
-        fig.legend(lines, linenames, loc='upper center',bbox_to_anchor = (0.4,1), prop={'size':10},ncol=len(linenames)/2)
+        fig.legend(lines, linenames, loc='upper center',bbox_to_anchor = (0.4,1), prop={'size':10},ncol=len(linenames)//2)
 #        fig.legend(lines, linenames, loc='upper right',bbox_to_anchor = bbox, prop={'size':8},ncol=ncol,title=ltitle)
     if title:
         ax.set_title("\n".join(wrap(title)))
-    axes = ax.get_axes()
-    axes.yaxis.grid(True,
+    axes = ax.get_yaxis()
+    axes.grid(True,
         linestyle='-',
         which='major',
         color='0.75'
     )
-    axes.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(num_yticks))
+    axes.set_major_locator(matplotlib.ticker.MaxNLocator(num_yticks))
     ax.set_axisbelow(True)
     ax.spines['right'].set_color('black')
     ax.spines['left'].set_color('black')
@@ -457,7 +457,7 @@ def draw_line(fname, data, xticks,
         savefig('../figs/' + 'legend.pdf')
         plt.close()
         fig = figure(figsize=((3.9, 0.6)))
-        fig.legend(lines, linenames,bbox_to_anchor = (1,1), prop={'size':10},frameon=False,ncol=len(linenames)/2)
+        fig.legend(lines, linenames,bbox_to_anchor = (1,1), prop={'size':10},frameon=False,ncol=len(linenames)//2)
         savefig('../figs/' + 'legend_half.pdf')
         plt.close()
 #        fig = figure(figsize=((4.2, 0.4)))
@@ -557,7 +557,7 @@ def draw_stack(data, xlabels, slabels, figname='stack', title=None, figsize=(8,
     htch = itertools.cycle(['','//','\\','-','\\\\','/'])
 
     for s in range(len(slabels)):
-        p = plt.bar(ind, data[s], color=clr.next(), hatch=htch.next(), bottom=bottom)
+        p = plt.bar(ind, data[s], color=next(clr), hatch=next(htch), bottom=bottom)
         plots = plots + (p,)
         bottom = [a + b for a,b in zip(bottom, data[s])]
 

scripts/helper.py

@@ -765,6 +765,8 @@ def get_summary(sfile,summary={}):
                 prog_tmp = {}
                 process_results(prog_tmp,results)
                 prog.append(prog_tmp)
+                if len(prog_tmp) == 0:
+                    continue
                 if int(prog_tmp["total_runtime"][0]) == 60:
                     if "post_warmup_txn_cnt" not in summary.keys():
                         summary["post_warmup_txn_cnt"] = [prog_tmp["txn_cnt"][0]]
@@ -1112,7 +1114,7 @@ def print_keys(result_dir="../results",keys=['txn_cnt']):
             elif len(summary_line) == 1:
                 res_line=summary_line[0][len('[summary]'):].strip()
             else:
-                assert false
+                assert False
             if res_line:
                 avail_keys = res_line.split(',')
                 keys_to_print=[k for k in avail_keys if k.split('=')[0] in keys]

scripts/paper_plots.py

@@ -133,8 +133,8 @@ def ppr_ycsb_partitions_plot(summary,summary_cl):
     v_name = "CC_ALG"
     x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":10000})
     tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_partitions",xlab="Partitions Accessed",new_cfgs=lst)
-    x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":12000})
-    tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_partitions_12k",xlab="Partitions Accessed",new_cfgs=lst)
+    #x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":12000})
+    #tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_partitions_12k",xlab="Partitions Accessed",new_cfgs=lst)
 
     nfmt,nexp = ycsb_partitions_distr()
     x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":10000})
@@ -157,7 +157,7 @@ def ppr_ycsb_partitions_abort_plot(summary,summary_cl):
 def ppr_ycsb_writes_plot(summary,summary_cl):
     from experiments import ycsb_writes   
     from helper import plot_prep
-    from plot_helper import tput
+    from plot_helper import tput,time_breakdown
     nfmt,nexp = ycsb_writes()
     x_name = "TXN_WRITE_PERC"
     v_name = "CC_ALG"
@@ -195,14 +195,14 @@ def ppr_ycsb_skew_plot(summary,summary_cl):
     x_name = "ZIPF_THETA"
     v_name = "CC_ALG"
 
-    x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":2})
-    tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_2",xlab="Zipf Theta",new_cfgs=lst,ylimit=120)
+    # x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":2})
+    # tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_2",xlab="Zipf Theta",new_cfgs=lst,ylimit=120)
 
-    x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":4})
-    tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_4",xlab="Zipf Theta",new_cfgs=lst,ylimit=120)
+    # x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":4})
+    # tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_4",xlab="Zipf Theta",new_cfgs=lst,ylimit=120)
 
-    x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":8})
-    tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_8",xlab="Zipf Theta",new_cfgs=lst,ylimit=120)
+    # x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":8})
+    # tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_8",xlab="Zipf Theta",new_cfgs=lst,ylimit=120)
 
     x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":16})
     tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_16",xlab="Skew Factor (Theta)",new_cfgs=lst)

scripts/parse_results.py

@@ -29,17 +29,17 @@ def get_summary(sfile):
     return results
 
 def process_results(results):
-	for r in results:
-		(name,val) = re.split('=',r)
-		val = float(val)
-		if name not in summary.keys():
-		    summary[name] = [val]
-		else:
-		    summary[name].append(val)
+    for r in results:
+        (name,val) = re.split('=',r)
+        val = float(val)
+        if name not in summary.keys():
+            summary[name] = [val]
+        else:
+            summary[name].append(val)
 
 
 if len(sys.argv) < 2:
-	sys.exit("Usage: %s [output-file-name]" % sys.argv[0])
+    sys.exit("Usage: %s [output-file-name]" % sys.argv[0])
 
 for arg in sys.argv[1:]:
     if not os.path.exists(arg):
@@ -50,8 +50,8 @@ def process_results(results):
 
 names.sort()
 
-print summary
-print names
+print(summary)
+print(names)
 
 s_avg = {}
 time_breakdown_total = 0
@@ -60,11 +60,11 @@ def process_results(results):
     s_avg[n] = avg(summary[n])
     if re.search('^time_',n):
         time_breakdown_total += s_avg[n]
-        print "{0}: {1}".format(n,s_avg[n] / s_avg['run_time'])
-
-print "% Runtime measured: {0}".format(time_breakdown_total / s_avg['run_time'])
-print "Compute time / txn: {0}".format( (s_avg['run_time'] - time_breakdown_total) / s_avg['txn_cnt'])
-print "Per-thread throughput: {0}".format(s_avg['txn_cnt'] / s_avg['run_time'])
-print "Throughput w/o waiting: {0}".format(s_avg['txn_cnt'] / (s_avg['run_time'] - s_avg['time_wait_lock'] - s_avg['time_wait_rem']))
-print "% Remote measured: {0}".format(( (0.000033 * s_avg['msg_sent'])+ s_avg['rtime_unpack'] + s_avg['rtime_proc']) / (s_avg['time_wait_lock'] + s_avg['time_wait_rem']))
-print s_avg
+        print("{0}: {1}").format(n,s_avg[n] / s_avg['run_time'])
+
+print("% Runtime measured: {0}").format(time_breakdown_total / s_avg['run_time'])
+print("Compute time / txn: {0}").format( (s_avg['run_time'] - time_breakdown_total) / s_avg['txn_cnt'])
+print("Per-thread throughput: {0}").format(s_avg['txn_cnt'] / s_avg['run_time'])
+print("Throughput w/o waiting: {0}").format(s_avg['txn_cnt'] / (s_avg['run_time'] - s_avg['time_wait_lock'] - s_avg['time_wait_rem']))
+print("% Remote measured: {0}").format(( (0.000033 * s_avg['msg_sent'])+ s_avg['rtime_unpack'] + s_avg['rtime_proc']) / (s_avg['time_wait_lock'] + s_avg['time_wait_rem']))
+print(s_avg)

scripts/plot.py

@@ -155,18 +155,18 @@
                                 r2 = get_summary(res_list[0],r2)
                     get_lstats(r)
                     get_lstats(r2)
-                    with open(p_sfile,'w') as f:
+                    with open(p_sfile,'wb') as f:
                         p = pickle.Pickler(f)
                         p.dump(r)
-                    with open(p_cfile,'w') as f:
+                    with open(p_cfile,'wb') as f:
                         p = pickle.Pickler(f)
                         p.dump(r2)
                 else:
-                    with open(p_sfile,'r') as f:
+                    with open(p_sfile,'rb') as f:
                         p = pickle.Unpickler(f)
                         r = p.load()
                         opened = True
-                    with open(p_cfile,'r') as f:
+                    with open(p_cfile,'rb') as f:
                         p = pickle.Unpickler(f)
                         r2 = p.load()
                         opened = True

scripts/plot_helper.py

@@ -499,6 +499,8 @@ def tput(xval,vval,summary,summary_cl,
         else:
             _xlab = xlab
     for v in vval:
+        print("============================================================")
+        print(v)
         if vname == 'MODE':
             mode_nice = {"NOCC_MODE":"No CC","NORMAL_MODE":"Serializable Execution","QRY_ONLY_MODE":"No Concurrency Control"}
             _v = mode_nice[v]
@@ -595,7 +597,7 @@ def tput(xval,vval,summary,summary_cl,
     print(_xval)
 
     # FIXME (Dana): MAAT --> OCC quick fix
-    print tpt.keys()
+    print (tpt.keys())
     if "MAAT" in tpt.keys():
         occ_tput = tpt["MAAT"]
         del tpt["MAAT"]
@@ -1289,6 +1291,9 @@ def time_breakdown(xval,summary,
     for x,i in zip(xval,range(len(xval))):
         if new_cfgs != {}:
             my_cfg_fmt = cfg_fmt + [xname]
+            print("==============================================================")
+            print(new_cfgs)
+            print("==============================================================")
             my_cfg = new_cfgs[(x,0)] + [x]
             my_cfg,my_cfg_fmt = apply_extras(my_cfg_fmt,my_cfg,extras,xname,'')
         else:

scripts/run_experiments.py

@@ -66,18 +66,17 @@
         cfgs = get_cfgs(fmt,e)
         if remote:
             cfgs["TPORT_TYPE"],cfgs["TPORT_TYPE_IPC"],cfgs["TPORT_PORT"]="\"tcp\"","false",7000
-
-        output_f = get_outfile_name(cfgs)
+        output_f = get_outfile_name(cfgs, fmt)
 
         # Check whether experiment has been already been run in this batch
         if skip:
             if len(glob.glob('{}*{}*.out'.format(result_dir,output_f))) > 0:
-                print "Experiment exists in results folder... skipping"
+                print ("Experiment exists in results folder... skipping")
                 continue
 
         output_dir = output_f + "/"
         output_f = output_f + strnow 
-        print output_f
+        print (output_f)
 
         f = open("config.h",'r');
         lines = f.readlines()
@@ -182,7 +181,16 @@
             else:
                 nnodes = cfgs["NODE_CNT"]
                 nclnodes = cfgs["CLIENT_NODE_CNT"]
+                if not isinstance(nnodes, int):
+                    nnodes = cfgs[nnodes]
+                if not isinstance(nclnodes, int):
+                    nclnodes = cfgs[nclnodes]
                 pids = []
+
+                with open("ifconfig.txt",'w') as f_ifcfg:
+                    for i in range(nnodes+nclnodes):
+                        f_ifcfg.write("172.31.39.16\n")
+
                 print("Deploying: {}".format(output_f))
                 for n in range(nnodes+nclnodes):
                     if n < nnodes:

statistics/stats.cpp

@@ -23,7 +23,7 @@
 #include "stats_array.h"
 #include <time.h>
 #include <sys/times.h>
-#include <sys/vtimes.h>
+//#include <sys/vtimes.h>
 
 void Stats_thd::init(uint64_t thd_id) {
   DEBUG_M("Stats_thd::init part_cnt alloc\n");

statistics/stats_array.cpp

@@ -22,7 +22,7 @@
 #include "work_queue.h"
 #include <time.h>
 #include <sys/times.h>
-#include <sys/vtimes.h>
+//#include <sys/vtimes.h>
 
 void StatsArr::quicksort(int low_idx, int high_idx) {
   int low = low_idx;

storage/index_btree.cpp

@@ -403,12 +403,13 @@ RC index_btree::insert_into_leaf(glob_param params, bt_node * leaf, idx_key_t ke
 		leaf->pointers[idx] = (void *) item;
 		return RCOK;
 	}
-    while (insertion_point < leaf->num_keys && leaf->keys[insertion_point] < key)
+    while (insertion_point < leaf->num_keys && leaf->keys[insertion_point] < key ){
         insertion_point++;
 	for (i = leaf->num_keys; i > insertion_point; i--) {
         leaf->keys[i] = leaf->keys[i - 1];
         leaf->pointers[i] = leaf->pointers[i - 1];
     }
+    }
     leaf->keys[insertion_point] = key;
     leaf->pointers[insertion_point] = (void *)item;
     leaf->num_keys++;
@@ -499,22 +500,22 @@ RC index_btree::insert_into_parent(
     bt_node * parent = left->parent;
 
     /* Case: new root. */
-    if (parent == NULL)
+    if (parent == NULL){
         return insert_into_new_root(params, left, key, right);
-
+    }
 	UInt32 insert_idx = 0;
 	while (parent->keys[insert_idx] < key && insert_idx < parent->num_keys)
 		insert_idx ++;
 	// the parent has enough space, just insert into it
     if (parent->num_keys < order - 1) {
-		for (UInt32 i = parent->num_keys-1; i >= insert_idx; i--) {
-			parent->keys[i + 1] = parent->keys[i];
-			parent->pointers[i+2] = parent->pointers[i+1];
-		}
-		parent->num_keys ++;
-		parent->keys[insert_idx] = key;
-		parent->pointers[insert_idx + 1] = right;
-		return RCOK;
+	    for (UInt32 i = parent->num_keys-1; i >= insert_idx; i--) {
+		    parent->keys[i + 1] = parent->keys[i];
+		    parent->pointers[i+2] = parent->pointers[i+1];
+	    }
+	    parent->num_keys ++;
+	    parent->keys[insert_idx] = key;
+	    parent->pointers[insert_idx + 1] = right;
+	    return RCOK;
 	}
 
     /* Harder case:  split a node in order