diff --git a/Makefile b/Makefile index 6526488b..0c1c1bf8 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,15 @@ -CC=g++ +CC=g++ -faligned-new CFLAGS=-Wall -g -gdwarf-3 -std=c++0x #CFLAGS += -fsanitize=address -fno-omit-frame-pointer JEMALLOC=./jemalloc-4.0.3 -NNMSG=./nanomsg-0.5-beta +NNMSG=./nanomsg-0.6-beta .SUFFIXES: .o .cpp .h SRC_DIRS = ./ ./benchmarks/ ./client/ ./concurrency_control/ ./storage/ ./transport/ ./system/ ./statistics/#./unit_tests/ -DEPS = -I. -I./benchmarks -I./client/ -I./concurrency_control -I./storage -I./transport -I./system -I./statistics -I$(JEMALLOC)/include #-I./unit_tests +DEPS = -I. -I./boost_1_79_0 -I./benchmarks -I./client/ -I./concurrency_control -I./storage -I./transport -I./system -I./statistics -I$(JEMALLOC)/include #-I./unit_tests -CFLAGS += $(DEPS) -D NOGRAPHITE=1 -Werror -Wno-sizeof-pointer-memaccess +CFLAGS += $(DEPS) -D NOGRAPHITE=1 -Wno-sizeof-pointer-memaccess LDFLAGS = -Wall -L. -L$(NNMSG) -L$(JEMALLOC)/lib -Wl,-rpath,$(JEMALLOC)/lib -pthread -gdwarf-3 -lrt -std=c++0x #LDFLAGS = -Wall -L. -L$(NNMSG) -L$(JEMALLOC)/lib -Wl,-rpath,$(JEMALLOC)/lib -pthread -gdwarf-3 -lrt -std=c++11 LDFLAGS += $(CFLAGS) diff --git a/README.md b/README.md index 6fd4ea33..250e8165 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,20 @@ This testbed is based on the DBx1000 system, whose concurrency control scalabili Xiangyao Yu, George Bezerra, Andrew Pavlo, Srinivas Devadas, Michael Stonebraker http://voltdb.com/downloads/datasheets_collateral/vdb_whitepaper_staring_into_the_abyss.pdf -Build & Test +Setup ------------ +DDBMS has three dependences that need to be installed: + +* [Boost](https://www.boost.org/) +* [jemalloc](https://github.com/jemalloc/jemalloc/releases/tag/4.0.3) +* [nanomsg](https://github.com/nanomsg/nanomsg/releases/tag/0.6-beta) + +To be able to make the code successfully there needs to be a file named obj. Run + + mkdir obj +Build & Test +------------ To build the database. make deps @@ -43,3 +54,20 @@ The DBMS can be run with ./runcl -nid[M] where N and M are the ID of a server and client, respectively + +For example, if you are running with a `THREAD_CNT` of 4 you would run + + ./rundb -nid0 + ./rundb -nid1 + ./runcl -nid2 + ./runcl -nid3 + +There is also the option to run scripts. From the scripts directory run + + python3 run_experiments -e [experiment] + +* List of available experiments can be found [here](https://github.com/mitdbg/deneva/blob/master/scripts/experiments.py) + +After running an experiment, the results can be plotted by running + + python3 plot.py [experiment] diff --git a/concurrency_control/row_lock.cpp b/concurrency_control/row_lock.cpp index e880ecc5..bbbd602c 100644 --- a/concurrency_control/row_lock.cpp +++ b/concurrency_control/row_lock.cpp @@ -206,10 +206,11 @@ RC Row_lock::lock_get(lock_t type, TxnManager * txn, uint64_t* &txnids, int &txn INC_STATS(txn->get_thd_id(),twopl_getlock_time,timespan); INC_STATS(txn->get_thd_id(),twopl_getlock_cnt,1); - if (g_central_man) + if (g_central_man){ glob_manager.release_row(_row); - else - pthread_mutex_unlock( latch ); + }else{ + pthread_mutex_unlock( latch ); + } return rc; diff --git a/scripts/draw.py b/scripts/draw.py index 0cba8744..74c2af61 100644 --- a/scripts/draw.py +++ b/scripts/draw.py @@ -384,7 +384,7 @@ def draw_line(fname, data, xticks, if logscale : ax.set_yscale('log') if logscalex: - ax.set_xscale('log',basex=base) + ax.set_xscale('log') n = 0 if xlabels != None : ax.set_xticklabels([x if i%2 else '' for x,i in zip(xlabels,range(len(xlabels)))]) @@ -432,17 +432,17 @@ def draw_line(fname, data, xticks, ax.get_xaxis().set_major_formatter(matplotlib.ticker.FormatStrFormatter('%d')) if legend : #fig.legend(lines, linenames, loc='upper right',bbox_to_anchor = (1,1), prop={'size':9}, ncol=ncol) - fig.legend(lines, linenames, loc='upper center',bbox_to_anchor = (0.4,1), prop={'size':10},ncol=len(linenames)/2) + fig.legend(lines, linenames, loc='upper center',bbox_to_anchor = (0.4,1), prop={'size':10},ncol=len(linenames)//2) # fig.legend(lines, linenames, loc='upper right',bbox_to_anchor = bbox, prop={'size':8},ncol=ncol,title=ltitle) if title: ax.set_title("\n".join(wrap(title))) - axes = ax.get_axes() - axes.yaxis.grid(True, + axes = ax.get_yaxis() + axes.grid(True, linestyle='-', which='major', color='0.75' ) - axes.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(num_yticks)) + axes.set_major_locator(matplotlib.ticker.MaxNLocator(num_yticks)) ax.set_axisbelow(True) ax.spines['right'].set_color('black') ax.spines['left'].set_color('black') @@ -457,7 +457,7 @@ def draw_line(fname, data, xticks, savefig('../figs/' + 'legend.pdf') plt.close() fig = figure(figsize=((3.9, 0.6))) - fig.legend(lines, linenames,bbox_to_anchor = (1,1), prop={'size':10},frameon=False,ncol=len(linenames)/2) + fig.legend(lines, linenames,bbox_to_anchor = (1,1), prop={'size':10},frameon=False,ncol=len(linenames)//2) savefig('../figs/' + 'legend_half.pdf') plt.close() # fig = figure(figsize=((4.2, 0.4))) @@ -557,7 +557,7 @@ def draw_stack(data, xlabels, slabels, figname='stack', title=None, figsize=(8, htch = itertools.cycle(['','//','\\','-','\\\\','/']) for s in range(len(slabels)): - p = plt.bar(ind, data[s], color=clr.next(), hatch=htch.next(), bottom=bottom) + p = plt.bar(ind, data[s], color=next(clr), hatch=next(htch), bottom=bottom) plots = plots + (p,) bottom = [a + b for a,b in zip(bottom, data[s])] diff --git a/scripts/helper.py b/scripts/helper.py index 6fb2867d..82713fe8 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -765,6 +765,8 @@ def get_summary(sfile,summary={}): prog_tmp = {} process_results(prog_tmp,results) prog.append(prog_tmp) + if len(prog_tmp) == 0: + continue if int(prog_tmp["total_runtime"][0]) == 60: if "post_warmup_txn_cnt" not in summary.keys(): summary["post_warmup_txn_cnt"] = [prog_tmp["txn_cnt"][0]] @@ -1112,7 +1114,7 @@ def print_keys(result_dir="../results",keys=['txn_cnt']): elif len(summary_line) == 1: res_line=summary_line[0][len('[summary]'):].strip() else: - assert false + assert False if res_line: avail_keys = res_line.split(',') keys_to_print=[k for k in avail_keys if k.split('=')[0] in keys] diff --git a/scripts/paper_plots.py b/scripts/paper_plots.py index b8e7796a..1d33d816 100644 --- a/scripts/paper_plots.py +++ b/scripts/paper_plots.py @@ -133,8 +133,8 @@ def ppr_ycsb_partitions_plot(summary,summary_cl): v_name = "CC_ALG" x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":10000}) tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_partitions",xlab="Partitions Accessed",new_cfgs=lst) - x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":12000}) - tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_partitions_12k",xlab="Partitions Accessed",new_cfgs=lst) + #x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":12000}) + #tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_partitions_12k",xlab="Partitions Accessed",new_cfgs=lst) nfmt,nexp = ycsb_partitions_distr() x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"MAX_TXN_IN_FLIGHT":10000}) @@ -157,7 +157,7 @@ def ppr_ycsb_partitions_abort_plot(summary,summary_cl): def ppr_ycsb_writes_plot(summary,summary_cl): from experiments import ycsb_writes from helper import plot_prep - from plot_helper import tput + from plot_helper import tput,time_breakdown nfmt,nexp = ycsb_writes() x_name = "TXN_WRITE_PERC" v_name = "CC_ALG" @@ -195,14 +195,14 @@ def ppr_ycsb_skew_plot(summary,summary_cl): x_name = "ZIPF_THETA" v_name = "CC_ALG" - x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":2}) - tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_2",xlab="Zipf Theta",new_cfgs=lst,ylimit=120) + # x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":2}) + # tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_2",xlab="Zipf Theta",new_cfgs=lst,ylimit=120) - x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":4}) - tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_4",xlab="Zipf Theta",new_cfgs=lst,ylimit=120) + # x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":4}) + # tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_4",xlab="Zipf Theta",new_cfgs=lst,ylimit=120) - x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":8}) - tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_8",xlab="Zipf Theta",new_cfgs=lst,ylimit=120) + # x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":8}) + # tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_8",xlab="Zipf Theta",new_cfgs=lst,ylimit=120) x_vals,v_vals,fmt,exp,lst = plot_prep(nexp,nfmt,x_name,v_name,constants={"NODE_CNT":16}) tput(x_vals,v_vals,summary,summary_cl,cfg_fmt=fmt,cfg=list(exp),xname=x_name,vname=v_name,title="",name="tput_ycsb_skew_16",xlab="Skew Factor (Theta)",new_cfgs=lst) diff --git a/scripts/parse_results.py b/scripts/parse_results.py index 3a7b237c..8a110977 100755 --- a/scripts/parse_results.py +++ b/scripts/parse_results.py @@ -29,17 +29,17 @@ def get_summary(sfile): return results def process_results(results): - for r in results: - (name,val) = re.split('=',r) - val = float(val) - if name not in summary.keys(): - summary[name] = [val] - else: - summary[name].append(val) + for r in results: + (name,val) = re.split('=',r) + val = float(val) + if name not in summary.keys(): + summary[name] = [val] + else: + summary[name].append(val) if len(sys.argv) < 2: - sys.exit("Usage: %s [output-file-name]" % sys.argv[0]) + sys.exit("Usage: %s [output-file-name]" % sys.argv[0]) for arg in sys.argv[1:]: if not os.path.exists(arg): @@ -50,8 +50,8 @@ def process_results(results): names.sort() -print summary -print names +print(summary) +print(names) s_avg = {} time_breakdown_total = 0 @@ -60,11 +60,11 @@ def process_results(results): s_avg[n] = avg(summary[n]) if re.search('^time_',n): time_breakdown_total += s_avg[n] - print "{0}: {1}".format(n,s_avg[n] / s_avg['run_time']) - -print "% Runtime measured: {0}".format(time_breakdown_total / s_avg['run_time']) -print "Compute time / txn: {0}".format( (s_avg['run_time'] - time_breakdown_total) / s_avg['txn_cnt']) -print "Per-thread throughput: {0}".format(s_avg['txn_cnt'] / s_avg['run_time']) -print "Throughput w/o waiting: {0}".format(s_avg['txn_cnt'] / (s_avg['run_time'] - s_avg['time_wait_lock'] - s_avg['time_wait_rem'])) -print "% Remote measured: {0}".format(( (0.000033 * s_avg['msg_sent'])+ s_avg['rtime_unpack'] + s_avg['rtime_proc']) / (s_avg['time_wait_lock'] + s_avg['time_wait_rem'])) -print s_avg + print("{0}: {1}").format(n,s_avg[n] / s_avg['run_time']) + +print("% Runtime measured: {0}").format(time_breakdown_total / s_avg['run_time']) +print("Compute time / txn: {0}").format( (s_avg['run_time'] - time_breakdown_total) / s_avg['txn_cnt']) +print("Per-thread throughput: {0}").format(s_avg['txn_cnt'] / s_avg['run_time']) +print("Throughput w/o waiting: {0}").format(s_avg['txn_cnt'] / (s_avg['run_time'] - s_avg['time_wait_lock'] - s_avg['time_wait_rem'])) +print("% Remote measured: {0}").format(( (0.000033 * s_avg['msg_sent'])+ s_avg['rtime_unpack'] + s_avg['rtime_proc']) / (s_avg['time_wait_lock'] + s_avg['time_wait_rem'])) +print(s_avg) diff --git a/scripts/plot.py b/scripts/plot.py index e328f5ee..06cfeb3e 100644 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -155,18 +155,18 @@ r2 = get_summary(res_list[0],r2) get_lstats(r) get_lstats(r2) - with open(p_sfile,'w') as f: + with open(p_sfile,'wb') as f: p = pickle.Pickler(f) p.dump(r) - with open(p_cfile,'w') as f: + with open(p_cfile,'wb') as f: p = pickle.Pickler(f) p.dump(r2) else: - with open(p_sfile,'r') as f: + with open(p_sfile,'rb') as f: p = pickle.Unpickler(f) r = p.load() opened = True - with open(p_cfile,'r') as f: + with open(p_cfile,'rb') as f: p = pickle.Unpickler(f) r2 = p.load() opened = True diff --git a/scripts/plot_helper.py b/scripts/plot_helper.py index 63f02ae7..b2829a00 100644 --- a/scripts/plot_helper.py +++ b/scripts/plot_helper.py @@ -499,6 +499,8 @@ def tput(xval,vval,summary,summary_cl, else: _xlab = xlab for v in vval: + print("============================================================") + print(v) if vname == 'MODE': mode_nice = {"NOCC_MODE":"No CC","NORMAL_MODE":"Serializable Execution","QRY_ONLY_MODE":"No Concurrency Control"} _v = mode_nice[v] @@ -595,7 +597,7 @@ def tput(xval,vval,summary,summary_cl, print(_xval) # FIXME (Dana): MAAT --> OCC quick fix - print tpt.keys() + print (tpt.keys()) if "MAAT" in tpt.keys(): occ_tput = tpt["MAAT"] del tpt["MAAT"] @@ -1289,6 +1291,9 @@ def time_breakdown(xval,summary, for x,i in zip(xval,range(len(xval))): if new_cfgs != {}: my_cfg_fmt = cfg_fmt + [xname] + print("==============================================================") + print(new_cfgs) + print("==============================================================") my_cfg = new_cfgs[(x,0)] + [x] my_cfg,my_cfg_fmt = apply_extras(my_cfg_fmt,my_cfg,extras,xname,'') else: diff --git a/scripts/run_experiments.py b/scripts/run_experiments.py index 9218885e..d60c9e49 100644 --- a/scripts/run_experiments.py +++ b/scripts/run_experiments.py @@ -66,18 +66,17 @@ cfgs = get_cfgs(fmt,e) if remote: cfgs["TPORT_TYPE"],cfgs["TPORT_TYPE_IPC"],cfgs["TPORT_PORT"]="\"tcp\"","false",7000 - - output_f = get_outfile_name(cfgs) + output_f = get_outfile_name(cfgs, fmt) # Check whether experiment has been already been run in this batch if skip: if len(glob.glob('{}*{}*.out'.format(result_dir,output_f))) > 0: - print "Experiment exists in results folder... skipping" + print ("Experiment exists in results folder... skipping") continue output_dir = output_f + "/" output_f = output_f + strnow - print output_f + print (output_f) f = open("config.h",'r'); lines = f.readlines() @@ -182,7 +181,16 @@ else: nnodes = cfgs["NODE_CNT"] nclnodes = cfgs["CLIENT_NODE_CNT"] + if not isinstance(nnodes, int): + nnodes = cfgs[nnodes] + if not isinstance(nclnodes, int): + nclnodes = cfgs[nclnodes] pids = [] + + with open("ifconfig.txt",'w') as f_ifcfg: + for i in range(nnodes+nclnodes): + f_ifcfg.write("172.31.39.16\n") + print("Deploying: {}".format(output_f)) for n in range(nnodes+nclnodes): if n < nnodes: diff --git a/statistics/stats.cpp b/statistics/stats.cpp index 296cf8e5..a92bd6a3 100644 --- a/statistics/stats.cpp +++ b/statistics/stats.cpp @@ -23,7 +23,7 @@ #include "stats_array.h" #include #include -#include +//#include void Stats_thd::init(uint64_t thd_id) { DEBUG_M("Stats_thd::init part_cnt alloc\n"); diff --git a/statistics/stats_array.cpp b/statistics/stats_array.cpp index 725a2196..7828bc09 100644 --- a/statistics/stats_array.cpp +++ b/statistics/stats_array.cpp @@ -22,7 +22,7 @@ #include "work_queue.h" #include #include -#include +//#include void StatsArr::quicksort(int low_idx, int high_idx) { int low = low_idx; diff --git a/storage/index_btree.cpp b/storage/index_btree.cpp index 6779da8a..63ff0456 100644 --- a/storage/index_btree.cpp +++ b/storage/index_btree.cpp @@ -403,12 +403,13 @@ RC index_btree::insert_into_leaf(glob_param params, bt_node * leaf, idx_key_t ke leaf->pointers[idx] = (void *) item; return RCOK; } - while (insertion_point < leaf->num_keys && leaf->keys[insertion_point] < key) + while (insertion_point < leaf->num_keys && leaf->keys[insertion_point] < key ){ insertion_point++; for (i = leaf->num_keys; i > insertion_point; i--) { leaf->keys[i] = leaf->keys[i - 1]; leaf->pointers[i] = leaf->pointers[i - 1]; } + } leaf->keys[insertion_point] = key; leaf->pointers[insertion_point] = (void *)item; leaf->num_keys++; @@ -499,22 +500,22 @@ RC index_btree::insert_into_parent( bt_node * parent = left->parent; /* Case: new root. */ - if (parent == NULL) + if (parent == NULL){ return insert_into_new_root(params, left, key, right); - + } UInt32 insert_idx = 0; while (parent->keys[insert_idx] < key && insert_idx < parent->num_keys) insert_idx ++; // the parent has enough space, just insert into it if (parent->num_keys < order - 1) { - for (UInt32 i = parent->num_keys-1; i >= insert_idx; i--) { - parent->keys[i + 1] = parent->keys[i]; - parent->pointers[i+2] = parent->pointers[i+1]; - } - parent->num_keys ++; - parent->keys[insert_idx] = key; - parent->pointers[insert_idx + 1] = right; - return RCOK; + for (UInt32 i = parent->num_keys-1; i >= insert_idx; i--) { + parent->keys[i + 1] = parent->keys[i]; + parent->pointers[i+2] = parent->pointers[i+1]; + } + parent->num_keys ++; + parent->keys[insert_idx] = key; + parent->pointers[insert_idx + 1] = right; + return RCOK; } /* Harder case: split a node in order diff --git a/system/sequencer.cpp b/system/sequencer.cpp index 6067e579..e06bb928 100644 --- a/system/sequencer.cpp +++ b/system/sequencer.cpp @@ -247,7 +247,7 @@ void Sequencer::process_txn( Message * msg,uint64_t thd_id, uint64_t early_start } else { cl_msg->recon = false; - en->list[id].seq_startts = last_time; + en->list[id].seq_startts = last_time_batch; } }