Prototype of new scheduler

.Doxyfile

@@ -811,7 +811,7 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                = ./scanner/main.cpp
+EXCLUDE                = ./scanner/main.cpp ./python/scannerpy/build
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -838,7 +838,7 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        =
+EXCLUDE_SYMBOLS        = scanner::internal
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

.clang-format

@@ -41,7 +41,7 @@ BreakStringLiterals: true
 ColumnLimit:     80
 CommentPragmas:  '^ IWYU pragma:'
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
-ConstructorInitializerIndentWidth: 4
+ConstructorInitializerIndentWidth: 2
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
 DerivePointerAlignment: false
@@ -89,6 +89,6 @@ SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
 Standard:        Auto
-TabWidth:        8
+TabWidth:        2
 UseTab:          Never
 ...

.gdbinit

@@ -0,0 +1 @@
+handle SIG40 nostop noprint

.gitignore

@@ -2,15 +2,21 @@
 # similar but **not quite** the same. If you add a file here, make sure that
 # the Docker build still works.
 
+dependencies.txt
+nvidia-docker-compose.yml
 .cache
 build
 **/*.pyc
 **/*.trace
 thirdparty
+!thirdparty/resources
 !thirdparty/CMakeLists.txt
 python/*.egg-info
-
+docker/**/deps.sh
+docker/**/thirdparty
 **/*.mp4
 **/*.mkv
+docker/*.log
 
-python/scannerpy/include
+python/scannerpy/include
+dist

.scanner.example.toml

@@ -11,4 +11,6 @@ scanner_path = "/opt/scanner"
 #     bucket = "gcs-bucket"
 
 [network]
-    master = "localhost:5001"
+    master = "localhost"
+    master_port = "5001"
+    worker_port = "5002"

.travis.yml

@@ -1,22 +1,66 @@
+language: python
+python:
+- '3.5'
 sudo: required
-
-language: cpp
-
 services:
 - docker
-
+addons:
+  apt:
+    packages:
+    - docker-ce
 env:
   global:
-    - DOCKER_REPO=scannerresearch/scanner
-    - DOCKER_EMAIL=wcrichto@cs.stanford.edu
-    - DOCKER_USER=wcrichto
-    - secure: "Jf7f8UvpJ8yRA/2nnxaRb9jGrzWZvbRVRxzQtmcx+ltBpflmLFbXB6o4sySGETn5YrJe1g4VNWvkFBZIdLY0n+v4AQ/D1eMWtJT+kZLHihVGcxB3G7SUSOyEUaN1DzdHbaRW4VCSXQJWgK48amxlwnNfXtbNkSpiqIifDURBeHK1UT9+w1FKuTDZmEXv+ubDKjXvv1DuPdTtnolItDfrdtKLh5iXgTRvgXFanmNvydIcHqfz+72qJZJRCdTwUrA8FPq7jKNndftX9y2/dKuA/Xd/cP818iR57Z2PJXeR1c6ZmQ0RBTIZ8pdwG6bfix+48a5+aWlPJ3friA0G2b8psdZ/rQMxBbEClTI+BfnuJnYaIW8JREqmOBACBL7/4eo/Jifv1+vl+VhWHSicS6mWg2khCNAMPWj3cIA/bZrRlTmNlC06GkUgnHar/HhovGrowbZCGOxN0CNWWcuCGeLYfvPgPiI4JRU8eq0VqVhhqXZyPuS/RT1FoqZ4mMfNZG+RSIKHU56g5M4elakp/bZdu9lRoS4hr9VFpvPaLorOTOXPa3J+JxmB9OIP+NLxMIGjUhYnSYfTJ9/+Opxg2lWUOdqXzPZsyPyI+4RuIyyPh1w6jN/xD3EqQC55u5JlFlJ1WuNoyhx16QcKu1Eytxd2JxU6LutMw9WgQET53Q3EOVk=" # DOCKER_PASS
-    - secure: "mJDNPmfRd3cyFNIhRg6TQo8ow5hOS+FC+DX6MLpM6Giv2nL1KkJD0fgnLqgiZeEKDOFHvGpGswGjYTAzNWYeocsneby88w58sNbueUE6PT74UqsbzrmvMhKh2xtEERkJFz5gKvdwc9a4nAvH5ejJW7OZ5OvSscR2vRBl4tUwTuA3czWbod6NwBC/sUrf0jTZhNgZWWWT1j2SLt379Q52Xmd5+ixAaSibctf8hCEMzE/lNsWprR1gO7H89eyLVRQc8VQh8msVtgo7QD/aapr+w96GDWWUAm6z5iNe8gLGZB9v4BtTm82XOV0iefNDfT0fwIhHrM6vqIdiiuMeYMteErT3rF55h0o8hJrP57lwT5u7hU/yxUhNTvtCdJTo8THJihXHbOT7LjxaF20SlJzjARSGZq9mYHeBgv9sGej2jF80iSn+xTAbalVFbc4hKumELnn/D7mGWRTjsGGTbTqNXvTQvxm/QjXmNusav5vKeRFwjbPGfb0iDSfhlZ7BHkR50OyskZ/R/69WjIFS9gBE2pvp4FlVpndKWkkkd4TVSgmnKCXxnQdeNS8We8NodvTBlTcvHFub8j2izKTZRU7FBXRgo9d+Gu+43nSRvjJUgwg/6D6Bou6boVHJo8ib9xzakVjojbJxehatj+dadpuVkG3tjULejPNg/qnjlJc10Pg=" # PYPI_PASS
-
+  - BUILD_TYPE=cpu
+  - DOCKER_REPO=scannerresearch/scanner
+  - DOCKER_TEST_REPO=scannerresearch/test-builds
+  - DOCKER_EMAIL=wcrichto@cs.stanford.edu
+  - DOCKER_USER=wcrichto
+  - secure: DAwB02uem8/aIRNfHPknzqZCCmwDlJoQq6r3qE1CyUqkM9VMu6mQa/RBYMBwp3Upny6oI6yBlYJqqVr1rjZzI4by0j0IVhuhQPp2G5DJ/mVZ5PQSJ0bxJtCzm29hHo8hi+EgIk7r7YTcHOr3SRmplf0vM0WFuJplW5fJ6J/RbJSPPpnlTOIgp0Ry9B4eowLhi4b//TbwmtzEX6j4yKxWnDTJ6cV9xRVN1FoNd15PvpiBkOZ3dVGWtdUZm9K3Q3IumeyuemAoTSuKU3r3aqMSamdLf7pmYjpzTrYDT9CMbbEVjzr8M3Y5wFyjQsIdFKzYM/0jRBlUbKaLMhz7a/9vlkplBI954ufOqHiGAx3Cdk4jfkJ3hyucCQqYuQwHxWMmru3lV6jIZf+rYn6UBomHeNN1AIiikZ1EjvNDwY8iwGZPJbcZ42gn2mTaUVxsmwMD757AalTOzZo94+pdFJDjWY6y6kPQlZbdj8AT14bBHf8x8zF8EsZrh/WQMcRgzEU3BcjDE3RIqNgK38Sv1OzqNVTYy57PXbEMKMqfpAQx8FTcx/7NXBRrStrQ24pRZJA+wvkUefpyWZnayyMPvttzB4106GCOWAjSdJEyYQTnZMrXMRX4Qf+NHx6KHlQcExxpww/hSvewsh0JRrvMol5HKXJaZqnF6ZTqxeQdLUDdn3g=
+  - secure: mJDNPmfRd3cyFNIhRg6TQo8ow5hOS+FC+DX6MLpM6Giv2nL1KkJD0fgnLqgiZeEKDOFHvGpGswGjYTAzNWYeocsneby88w58sNbueUE6PT74UqsbzrmvMhKh2xtEERkJFz5gKvdwc9a4nAvH5ejJW7OZ5OvSscR2vRBl4tUwTuA3czWbod6NwBC/sUrf0jTZhNgZWWWT1j2SLt379Q52Xmd5+ixAaSibctf8hCEMzE/lNsWprR1gO7H89eyLVRQc8VQh8msVtgo7QD/aapr+w96GDWWUAm6z5iNe8gLGZB9v4BtTm82XOV0iefNDfT0fwIhHrM6vqIdiiuMeYMteErT3rF55h0o8hJrP57lwT5u7hU/yxUhNTvtCdJTo8THJihXHbOT7LjxaF20SlJzjARSGZq9mYHeBgv9sGej2jF80iSn+xTAbalVFbc4hKumELnn/D7mGWRTjsGGTbTqNXvTQvxm/QjXmNusav5vKeRFwjbPGfb0iDSfhlZ7BHkR50OyskZ/R/69WjIFS9gBE2pvp4FlVpndKWkkkd4TVSgmnKCXxnQdeNS8We8NodvTBlTcvHFub8j2izKTZRU7FBXRgo9d+Gu+43nSRvjJUgwg/6D6Bou6boVHJo8ib9xzakVjojbJxehatj+dadpuVkG3tjULejPNg/qnjlJc10Pg=
+  - secure: J8Em68qmB/q+u9PEyNujfUN0zbUEKVgjylBNHctL5Bo/9HgQs7yoYBxuEhoY0f95lYJYdsrJ9sOoZ7pMBgCFi9UY8+Gv4f9RfpTVYsp+lav0bc1MNI3Ct8wgEO3uCbIhUSZQFNcW1YsNv2lruyyKxQpQhBJLuS4wR5UkZ+AhjnWKL6S3916zxhwSwKkFxtYDD8rEkfBlymgd9kEqygNzencp9v2JosV90bczUF1wDaDpCLczxjKlwODZWa2milKC3kWaJ5lvee42at0ik3jLrNoCaEzO3u2tMtY3sbjXOVAXy3qROz3mCjU9mFO4Zu22XW4qfhbfFsXR3ht8BSuQ2JqIsgpdWz7gvkYHB6I08RmPgyEYO2BXu4/lVGx+60JUMAJ8teasOVldId6MJ8CzjU6Bl57yRNDZQFpexK+HprXdFtj7O4qfrcp63jPj08r2x1ZxdTMk40PLJ9x8vpIJ3Drb8xegf8I4osQuAVPgBKnZhWGqfAYL6KVr+27U+4PqnhA6VfuJoh4JjfF+5Uxz3CubEKwNYtWgOu2J7fjHCCcF6wjSr+XoyNmDoABL6yvbs65F+XoP9OP+VCgKNsyz/FksXHvmTVIggePgXd5oZzh+EottaKD19EE9FFMeenqIkQ8rDmxogt4SRW9HJIOa6ocQD0JQXKOBM4gynXBfgHw=
 install:
-- sudo apt-get install -y doxygen graphviz python-pip
-- pip install doxypypy twine
-
-script: ./scripts/travis-build.sh
-
-after_success: ./scripts/travis-publish.sh
+- sudo apt-get update
+- sudo apt-get install -y doxygen graphviz
+- pip3 install requests[security] --upgrade
+jobs:
+  include:
+    - stage: "Test build"
+      script: "travis_retry ./scripts/travis-build.sh"
+    # - stage: "Test build"
+    #   language: generic
+    #   services:
+    #     addons:
+    #   install:
+    #     pip3 install requests[security] --upgrade
+    #   script: "travis_retry ./scripts/travis-osx-build.sh"
+    #   os: osx
+    - stage: test
+      script: "travis_retry ./scripts/travis-test.sh"
+      env: TEST_TYPE=cpp
+    - stage: test
+      script: "travis_retry ./scripts/travis-test.sh"
+      env: TEST_TYPE=tutorials
+    - stage: test
+      script: "travis_retry ./scripts/travis-test.sh"
+      env: TEST_TYPE=integration
+    - stage: build
+      script: "travis_retry ./scripts/travis-build.sh"
+      after_success: "travis_retry ./scripts/travis-publish.sh"
+      env:
+        - BUILD_TYPE=cpu
+    - stage: build
+      script: "travis_retry ./scripts/travis-build.sh"
+      env:
+        - BUILD_TYPE=gpu-8.0-cudnn6
+    - stage: build
+      script: "travis_retry ./scripts/travis-build.sh"
+      env:
+        - BUILD_TYPE=gpu-8.0-cudnn7
+    - stage: build
+      script: "travis_retry ./scripts/travis-build.sh"
+      env:
+        - BUILD_TYPE=gpu-9.0-cudnn7
+    - stage: build
+      script: "travis_retry ./scripts/travis-build.sh"
+      env:
+        - BUILD_TYPE=gpu-9.1-cudnn7

.travis/travisci_rsa_brew.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw2maYHWQ6cdBi9UJ2Dzv4okW/bw0pmZAKAFZvXOjZ8rRcrhgu8pK289t42Hwso9dsCROJxxVegDTZeOh8jyuQqKziBXu1MqN/3+KuRWJfw3QGwOQNOuwPkBOEn+TZpkgiyz2GMDfvy1WseWxgKzadY2C/R4lHafnmc4Dr/6k2uNST3s73CAyHlGpORItlqZ+oc/IX74qz27PsjMjZsjarCEHDb3UWlsG95D1ymMWrwXUpx6/azyuAJEiWJp/UtdfL8oJVec9vXqK9pzI6IykLdFm0y72FYXtMwBDARugDGpqTpO1xmba0kSAoMe5Ip24nbNHB6gIHL8dnAD9qrbzF apoms@AlexandersMBP2.lan

CMakeLists.txt

@@ -19,8 +19,6 @@ project(Scanner)
 ###### Config options #####
 option(BUILD_CUDA "" ON)
 option(BUILD_TESTS "" ON)
-option(BUILD_SERVER "" OFF)
-option(BUILD_EXAMPLES "" ON)
 option(ENABLE_PROFILING "" OFF)
 
 if (BUILD_TESTS)
@@ -47,9 +45,14 @@ set(GLOBAL_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
 set(THIRDPARTY_SOURCE_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 set(THIRDPARTY_OUTPUT_PATH "${THIRDPARTY_SOURCE_DIR}/build/bin")
 
-if (NOT CMAKE_BUILD_TYPE)
+if(APPLE)
+  message(STATUS "Non-debug builds fail on MacOS. Setting to debug.")
+  set(CMAKE_BUILD_TYPE "Debug")
+elseif(UNIX)
+  if (NOT CMAKE_BUILD_TYPE)
     message(STATUS "No build type selected, defaulting to Release")
     set(CMAKE_BUILD_TYPE "Release")
+  endif()
 endif()
 
 if(CMAKE_BUILD_TYPE MATCHES Debug)
@@ -66,101 +69,14 @@ if (ENABLE_PROFILING)
   add_definitions(-DSCANNER_PROFILING)
 endif()
 
-###### Optional Dependencies #######
-if (BUILD_CUDA)
-  find_package(CUDA REQUIRED)
-  add_definitions(-DHAVE_CUDA)
-  include_directories(${CUDA_INCLUDE_DIRS})
-  if(COMPILER_SUPPORTS_CXX1Y)
-    set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++11")
-  endif()
-endif()
-
-if (BUILD_CUDA)
-  add_library(scanner_halide scanner/engine/halide_context.cpp)
-endif()
-
-###### Required Dependencies #######
-find_package(SaneProtobuf REQUIRED)
-find_package(FFmpeg REQUIRED)
-find_package(LibLZMA REQUIRED)
-find_package(OpenSSL REQUIRED)
-find_package(BZip2 REQUIRED)
-find_package(Boost COMPONENTS thread program_options regex python REQUIRED)
-find_package(GFlags REQUIRED)
-find_package(Glog REQUIRED)
-find_package(GoogleTest REQUIRED)
-find_package(CURL REQUIRED)
-find_package(Iconv REQUIRED)
-find_package(Storehouse REQUIRED CONFIG
-  PATHS "thirdparty/build/bin/storehouse")
-find_package(TinyToml REQUIRED)
-find_package(PythonLibs 2.7 EXACT REQUIRED)
-
-set(GTEST_INCLUDE_DIRS
-  "${THIRDPARTY_OUTPUT_PATH}/googletest/include")
-set(GTEST_LIBRARIES
-  "${THIRDPARTY_OUTPUT_PATH}/googletest/lib/libgtest.a")
-set(GTEST_LIB_MAIN
-  "${THIRDPARTY_OUTPUT_PATH}/googletest/lib/libgtest_main.a")
-
-set(SCANNER_LIBRARIES
-  "${PROTOBUF_LIBRARY}"
-  "${STOREHOUSE_LIBRARIES}"
-  "${FFMPEG_LIBRARIES}"
-  "-L/opt/ffmpeg-3.2.2/lib"
-  "-lswscale"
-  "${LIBLZMA_LIBRARIES}"
-  "${OPENSSL_LIBRARIES}"
-  "${BZIP2_LIBRARIES}"
-  "${PROXYGEN_LIBRARIES}"
-  "${FOLLY_LIBRARIES}"
-  "${Boost_LIBRARIES}"
-  "${GFLAGS_LIBRARIES}"
-  "${GLOG_LIBRARIES}"
-  "${CURL_LIBRARIES}"
-  "${ICONV_LIBRARIES}"
-  "${SCANNER_LIBRARIES}"
-  "${PYTHON_LIBRARIES}"
-  "-ljpeg"
-  "-lz"
-  "-ldl"
-  "-lgrpc++_unsecure -lgrpc -lgpr")
-
-include_directories(
-  "."
-  "${CMAKE_CURRENT_BINARY_DIR}" # for protobuf generated files
-  "${PROTOBUF_INCLUDE_DIRS}"
-  "${FFMPEG_INCLUDE_DIR}"
-  "${TINYTOML_INCLUDE_DIR}"
-  "${STOREHOUSE_INCLUDE_DIRS}"
-  "${OPENSSL_INCLUDE_DIR}"
-  "${Boost_INCLUDE_DIRS}"
-  "${GLOG_INCLUDE_DIRS}"
-  "${LIBLZMA_INCLUDE_DIRS}"
-  "${PYTHON_INCLUDE_DIRS}")
-
-if (BUILD_TESTS)
-  include_directories("${GTEST_INCLUDE_DIRS}")
-endif()
-
-if (BUILD_CUDA)
-  list(APPEND SCANNER_LIBRARIES
-    util_cuda
-    "${CUDA_LIBRARIES}"
-    "/usr/lib/x86_64-linux-gnu/libnvcuvid.so"
-    "-lcuda")
-endif()
-
-if (APPLE)
-  include_directories(
-    "/System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers/")
-elseif()
-endif()
+include(cmake/Dependencies.cmake)
 
 ###### Project code #######
 set(PROTO_FILES
   scanner/metadata.proto
+  scanner/source_args.proto
+  scanner/sink_args.proto
+  scanner/sampler_args.proto
   scanner/types.proto)
 
 set(GRPC_PROTO_FILES
@@ -200,6 +116,7 @@ add_library(scanner SHARED
   $<TARGET_OBJECTS:engine>
   $<TARGET_OBJECTS:video>
   $<TARGET_OBJECTS:util>
+  scanner/util/halide_context.cpp
   ${PROTO_SRCS}
   ${GRPC_PROTO_SRCS}
   ${STRUCK_SOURCES}
@@ -236,10 +153,6 @@ endforeach()
 
 add_subdirectory(stdlib)
 
-if (BUILD_EXAMPLES)
-  add_subdirectory(examples)
-endif()
-
 if (BUILD_TESTS)
   add_subdirectory(tests)
 endif()