Bitmaps/mod bit sets (stanford-futuredata#265)

* copied code from old branch to new * coded hybrid system * removed Arrays.sort and manually sorted * fixed null ptr bug * copied code from old branch to new * coded hybrid system * removed Arrays.sort and manually sorted * fixed null ptr bug * copied code from old branch to new * coded hybrid system * removed Arrays.sort and manually sorted * updated threshold * creating synthetic datasets * testing roaring bitmaps * Debugging options * Hybrid Apriori System * updated Dockerfile * Support threshold bug in AttributeEncoder fixed * BitMap -> Bitmap * Support threshold bug in AttributeEncoder fixed for all-bitmaps * Simplified hybrid system * Better cardinalityThreshold * hacky one-to-one and many-to-one strict FDs * Infrastructure for bitmaps cost model * Experimental bitmap cost model * Revised andCardinalityCost * Replacing RoaringBitmaps with BitSets * Don't materialize ands * Properly treat multiple same-cardinality columns * Per-instance bitmap decision * Better threshold application * Bitmap Ratio Threshold now a configurable parameter * Generalization of encodeAttributesWithSupport * Revert "Generalization of encodeAttributesWithSupport" This reverts commit 7b2e61d. * Better timing * Documentation * AttributeEncoder documentation * AttributeEncoder documentation * Update gitignore * Merge * Address Firas comments * Update .travis.yml Co-authored-by: Firas Abuzaid <[email protected]> Co-authored-by: Sahaana Suri <[email protected]>
ahshahid · Jan 10, 2020 · ae7b655 · ae7b655
1 parent 19d70ea
commit ae7b655
Show file tree

Hide file tree

Showing 23 changed files with 1,616 additions and 188 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,11 @@
+data/
+conf/
 *.class
 .classpath
 .project
 .settings/*
 *.log
+*.csv
 
 # Mobile Tools for Java (J2ME)
 .mtj.tmp/

diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,5 @@
 language: java
-jdk: oraclejdk8
+jdk: openjdk11
 install: mvn install -DskipTests=true -Dgpg.skip=true -Dmaven.javadoc.skip=true -B -V
 notifications:
   slack: stanford-futuredata:qmO6Keu8ifOyXHsmSQ97CeLH

diff --git a/bin/batch.sh b/bin/batch.sh
diff --git a/bin/cli.sh b/bin/cli.sh
diff --git a/bin/frontend.sh b/bin/frontend.sh
diff --git a/bin/macrobase-sql b/bin/macrobase-sql
diff --git a/bin/server.sh b/bin/server.sh
diff --git a/bin/streaming.sh b/bin/streaming.sh
diff --git a/build.sh b/build.sh
diff --git a/core/demo/query.sh b/core/demo/query.sh
diff --git a/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/BasicBatchPipeline.java b/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/BasicBatchPipeline.java
@@ -33,6 +33,7 @@ public class BasicBatchPipeline implements Pipeline {
     private boolean pctileLow;
     private String predicateStr;
     private int numThreads;
+    private int bitmapRatioThreshold;
 
     private String summarizerType;
     private List<String> attributes;
@@ -41,6 +42,9 @@ public class BasicBatchPipeline implements Pipeline {
     private double minRiskRatio;
     private double meanShiftRatio;
 
+    private boolean useFDs;
+    private int[] functionalDependencies;
+
 
     public BasicBatchPipeline (PipelineConfig conf) {
         inputURI = conf.get("inputURI");
@@ -71,6 +75,22 @@ public BasicBatchPipeline (PipelineConfig conf) {
         minRiskRatio = conf.get("minRatioMetric", 3.0);
         minSupport = conf.get("minSupport", 0.01);
         numThreads = conf.get("numThreads", Runtime.getRuntime().availableProcessors());
+        bitmapRatioThreshold = conf.get("bitmapRatioThreshold", 256);
+
+
+        //if FDs are behind used, parse them into bitmaps. For now, all FDs must be in the first 31 attributes
+        useFDs = conf.get("useFDs", false);
+        if (useFDs) {
+            ArrayList<ArrayList<Integer>> rawDependencies = conf.get("functionalDependencies");
+            functionalDependencies = new int[attributes.size()];
+            for (ArrayList<Integer> dependency : rawDependencies) {
+                for (int i : dependency) {
+                    for (int j : dependency) {
+                        if (i != j) functionalDependencies[i] |= (1 << j);
+                    }
+                }
+            }
+        }
         meanColumn = Optional.ofNullable(conf.get("meanColumn"));
         meanShiftRatio = conf.get("meanShiftRatio", 1.0);
     }
@@ -131,7 +151,10 @@ public BatchSummarizer getSummarizer(String outlierColumnName) throws MacroBaseE
                 summarizer.setAttributes(attributes);
                 summarizer.setMinSupport(minSupport);
                 summarizer.setMinRatioMetric(minRiskRatio);
+                summarizer.setBitmapRatioThreshold(bitmapRatioThreshold);
                 summarizer.setNumThreads(numThreads);
+                summarizer.setFDUsage(useFDs);
+                summarizer.setFDValues(functionalDependencies);
                 return summarizer;
             }
             case "countmeanshift": {

diff --git a/lib/genCP.sh b/lib/genCP.sh
diff --git a/lib/src/main/java/edu/stanford/futuredata/macrobase/analysis/summary/BatchSummarizer.java b/lib/src/main/java/edu/stanford/futuredata/macrobase/analysis/summary/BatchSummarizer.java
@@ -23,6 +23,8 @@ public abstract class BatchSummarizer implements Operator<DataFrame, Explanation
     protected int numThreads = Runtime.getRuntime().availableProcessors();
     protected String ratioMetric = "global_ratio";
     protected int maxOrder = 3;
+    protected boolean useFDs = false;
+    protected int[] functionalDependencies;
 
     /**
      * Adjust this to tune the significance (e.g. number of rows affected) of the results returned.
@@ -83,4 +85,14 @@ public BatchSummarizer setMaxOrder(final int maxOrder) throws MacroBaseException
         this.maxOrder = maxOrder;
         return this;
     }
+
+    public BatchSummarizer setFDUsage(final boolean useFDs) {
+        this.useFDs = useFDs;
+        return this;
+    }
+
+    public BatchSummarizer setFDValues(final int[] functionalDependencies) {
+        this.functionalDependencies = functionalDependencies;
+        return this;
+    }
 }
diff --git a/.../main/java/edu/stanford/futuredata/macrobase/analysis/summary/aplinear/APLSummarizer.java b/.../main/java/edu/stanford/futuredata/macrobase/analysis/summary/aplinear/APLSummarizer.java
@@ -1,14 +1,15 @@
 package edu.stanford.futuredata.macrobase.analysis.summary.aplinear;
 
 import edu.stanford.futuredata.macrobase.analysis.summary.BatchSummarizer;
-import edu.stanford.futuredata.macrobase.analysis.summary.util.AttributeEncoder;
 import edu.stanford.futuredata.macrobase.analysis.summary.util.qualitymetrics.AggregationOp;
 import edu.stanford.futuredata.macrobase.analysis.summary.util.qualitymetrics.QualityMetric;
+import edu.stanford.futuredata.macrobase.analysis.summary.util.AttributeEncoder;
 import edu.stanford.futuredata.macrobase.datamodel.DataFrame;
-import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.List;
+
 /**
  * Generic summarizer superclass that can be customized with
  * different quality metrics and input sources. Subclasses are responsible
@@ -21,10 +22,11 @@ public abstract class APLSummarizer extends BatchSummarizer {
     APrioriLinear aplKernel;
     List<QualityMetric> qualityMetricList;
     List<Double> thresholds;
-    private double[][] globalAggregateCols;
+    private double[][] globalAggregateCols = null;
 
     protected long numEvents = 0;
     protected long numOutliers = 0;
+    protected int bitmapRatioThreshold = 256;
 
     public abstract List<String> getAggregateNames();
     public abstract AggregationOp[] getAggregationOps();
@@ -59,7 +61,7 @@ public void process(DataFrame input) throws Exception {
         int[][] encoded = getEncoded(input.getStringColsByName(attributes), input);
         long elapsed = System.currentTimeMillis() - startTime;
         log.info("Encoded in: {} ms", elapsed);
-        log.info("Distinct values encoded: {}", encoder.getNextKey() - 1);
+        log.info("Encoded Categories: {}", encoder.getNextKey() - 1);
 
         thresholds = getThresholds();
         qualityMetricList = getQualityMetricList();
@@ -80,7 +82,10 @@ public void process(DataFrame input) throws Exception {
                 numThreads,
                 encoder.getBitmap(),
                 encoder.getOutlierList(),
-                encoder.getIsBitmapEncodedArray()
+                encoder.getColCardinalities(),
+                useFDs,
+                functionalDependencies,
+                bitmapRatioThreshold
         );
         log.info("Number of results: {}", aplResults.size());
         numOutliers = (long)getNumberOutliers(aggregateColumns);
@@ -99,8 +104,14 @@ public APLExplanation getResults() {
         return explanation;
     }
 
+    public void setBitmapRatioThreshold(int bitmapRatioThreshold) {
+        this.bitmapRatioThreshold = bitmapRatioThreshold;
+    }
+
     public void setGlobalAggregateCols(double[][] globalAggregateCols) {
         this.globalAggregateCols = globalAggregateCols;
     }
 
+
+
 }