added feature to increase max points returned by query

cmd/tsbs_run_queries_siridb/main.go

@@ -26,6 +26,7 @@ var (
 	dbPass       string
 	showExplain  bool
 	scale        uint64
+	queryLimit   uint64
 )
 
 // Global vars:
@@ -45,6 +46,7 @@ func init() {
 	flag.StringVar(&dbPass, "dbpass", "siri", "Password to enter SiriDB")
 	flag.StringVar(&hosts, "hosts", "localhost:9000", "Comma separated list of SiriDB hosts in a cluster.")
 	flag.Uint64Var(&scale, "scale", 8, "Scaling variable (Must be the equal to the scalevar used for data generation).")
+	flag.Uint64Var(&queryLimit, "query-limit", 1000000, "Changes the maximum points which can be returned by a select query.")
 	flag.IntVar(&writeTimeout, "write-timeout", 10, "Write timeout.")
 	flag.BoolVar(&showExplain, "show-explain", false, "Print out the EXPLAIN output for sample query")
 
@@ -78,6 +80,7 @@ func init() {
 
 func main() {
 	siridbConnector.Connect()
+	ChangeQueryLimit()
 	CreateGroups()
 
 	runner.Run(&query.SiriDBPool, newProcessor)
@@ -96,6 +99,22 @@ type processor struct {
 
 func newProcessor() query.Processor { return &processor{} }
 
+// Changes the maximum points which can be returned by a select query. The default
+// and recommended value is set to one million points. This value is chosen to
+// prevent a single query for taking to much memory and ensures SiriDB can respond
+// to almost any query in a reasonable amount of time.
+func ChangeQueryLimit() {
+	qry := fmt.Sprintf("alter database set select_points_limit %d", queryLimit)
+
+	if siridbConnector.IsConnected() {
+		if _, err := siridbConnector.Query(qry, uint16(writeTimeout)); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		log.Fatal("not even a single server is connected...")
+	}
+}
+
 // CreateGroups makes groups representing regular expression to enhance performance
 func CreateGroups() {
 	created := true

docs/siridb.md

@@ -66,6 +66,9 @@ A single host name or comma separated list of two host names for the SiriDB serv
 #### `-scale` (type: `uint64`, default: `8`)
 The scale is important for creating the right groups within SiriDB. It is the number of hosts for which data has been generated. So it should be the same as the scale (number of hosts) of the inserted data. This number is used to create a group for every host.
 
+#### `-query-limit` (type: `uint64`, default: `1000000`)
+The query limit changes the maximum points which can be returned by a select query. The default and recommended value is set to one million points. This value is chosen to prevent a single query for taking to much memory and ensures SiriDB can respond to almost any query in a reasonable amount of time. But in case of a large number of hosts it might be needed to increase the query-limit.
+
 #### `-write-timeout` (type: `int`, default: `10`)
 Length of the timeout for writes.
 

scripts/load_siridb.sh

@@ -17,7 +17,7 @@ optimize_interval = 9
 heartbeat_interval = 30
 default_db_path = /tmp/siridb/dbpath
 max_open_files = 512
-enable_shard_compression = 0
+enable_shard_compression = 1
 enable_pipe_support = 0
 buffer_sync_interval = 500
 EOT

scripts/run_queries_siridb.sh

@@ -13,6 +13,9 @@ fi
 # Scale necessary to create groups in siridb
 SCALE=${SCALE:-"8"}
 
+# Changes the maximum points which can be returned by a select query.
+QUERY_LIMIT=${QUERY_LIMIT:-"1000000"}
+
 # Queries folder
 BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"}
 
@@ -57,6 +60,7 @@ function run_file()
             -max-queries $MAX_QUERIES \
             -workers $NUM_WORKERS \
             -scale $SCALE \
+            -query-limit $QUERY_LIMIT \
         | tee $OUT_FULL_FILE_NAME
 }