Cleanup files from misc (faucetsdn#464)

SpinlockLabs · Jun 8, 2020 · 8510b81 · 8510b81
1 parent 0356852
commit 8510b81
Show file tree

Hide file tree

Showing 115 changed files with 119 additions and 117 deletions.
diff --git a/README.md b/README.md
@@ -43,24 +43,30 @@ The top-level DAQ folders correspond to the following structure:
 * `bin/`: System setup and management commands.
 * _`build/`_: Dynamically created directory for build logs.
 * `cmd/`: Primary commands for running DAQ testing.
+* `config/`: System and module configuration files.
 * `daq/`: Python source for DAQ runtime.
 * `docker/`: Docker build files for DAQ components and tests.
 * `docs/`: Documentation.
+* `etc/`: Version files and external switch setup files.
 * _`faucet/`_: Dynamically downloaded version of SDN controller.
+* _`forch/`_: Dynamically downloaded version of SDN orchestrator.
 * `firebase/`: Hosted pages and functions for web dashboard.
 * `functions/`: Additional Cloud Functions for data processing.
 * _`inst/`_: Install directory for specific runtime contents.
 * **`local/`**: Local setup and config information.
 * _`mininet/`_: Local version of the mininet virtual network host framework.
-* `misc/`: Miscellaneous support files.
 * `mudacl/`: Utilities for managing and testing MUD network files.
 * `mud_files/`: Examples and prototype device MUD files.
 * _`out/`_: Misc transitory output files.
+* `proto/`: Protobuf related files.
+* _`protoc-gen-doc/`_: Dynamically downloaded Protobuf documentation generator.
 * `pubber/`: Sample code for generating cloud-ingest traffic.
+* `resources/`: Templates for reports, module configs, test-site definitions.
 * `schemas/`: Device/cloud data exchange schemas.
 * `subset/`: Subsets of device tests (e.g. penetration tests).
 * `testing/`: Scripts for system continuous integration testing.
 * `topology/`: Network topology setups.
+* `utils/`: Helper utilities.
 * `validator/`: Tools for validating data exchange schemas.
 * _`venv/`_: Dynamically downloaded python virtual environment files.
 

diff --git a/bin/automated_run b/bin/automated_run
@@ -17,7 +17,7 @@ PIDFILE=inst/daq.pid
 SYSCONF=local/system.conf
 
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 if [ "$1" == debug ]; then
     auto_args+=" daq_loglevel=debug"

diff --git a/bin/build_hash b/bin/build_hash
@@ -10,7 +10,7 @@ build_built=.build_built
 faucet_version=$(cd faucet; git rev-list -n 1 HEAD)
 echo "$faucet_version  faucet/HEAD" > $build_files
 
-find misc/ docker/ subset/ utils/ -type f | sort | xargs sha1sum >> $build_files
+find docker/ subset/ utils/ -type f | sort | xargs sha1sum >> $build_files
 build_hash=`cat $build_files | sha256sum | awk '{print $1}'`
 
 if [ "$1" == check ]; then

diff --git a/bin/combine_reports_from_date_range b/bin/combine_reports_from_date_range
@@ -3,6 +3,6 @@
 ROOT=$(dirname $0)/..
 cd $ROOT
 
-source misc/config_base.sh 
+source etc/config_base.sh
 
 PYTHONPATH=daq python3 bin/python/combine_reports_from_date_range.py $conf_file $@ 
diff --git a/bin/docker_build b/bin/docker_build
@@ -5,7 +5,7 @@ set -o pipefail
 
 ROOT=$(dirname $0)/..
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 build_root=build
 sudo rm -rf $build_root .build_hash .build_files

diff --git a/bin/docker_build_files b/bin/docker_build_files
@@ -27,7 +27,7 @@ echo > $build_list_file
         true
     }
 
-    include ${host_tests:-misc/host_tests.conf}
+    include ${host_tests:-config/modules/host.conf}
 )
 cat $build_list_file
 rm -f $build_list_file
diff --git a/misc/dump_network.sh → bin/dump_network b/misc/dump_network.sh → bin/dump_network
diff --git a/bin/external_ovs b/bin/external_ovs
@@ -2,7 +2,7 @@
 
 ROOT=$(realpath $(dirname $0)/..)
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 ext_intf=$switch_setup_data_intf
 ext_dpid=$switch_setup_of_dpid

diff --git a/bin/generate_topology b/bin/generate_topology
@@ -2,7 +2,7 @@
 
 ROOT=$(realpath $(dirname $0)/..)
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 FAUCET=$(realpath $ROOT/faucet)
 MININET=$(realpath $ROOT/mininet)

diff --git a/bin/install_daq b/bin/install_daq
@@ -1,8 +1,8 @@
 #!/bin/bash -e
 
 ROOT=$(dirname $(realpath $0))/..
-sudo cp misc/daq.service /etc/systemd/system/
+sudo cp etc/daq.service /etc/systemd/system/
 sudo sed -i "s#%ROOT%#$ROOT#" /etc/systemd/system/daq.service 
 
 sudo systemctl enable daq.service
-sudo systemctl start daq.service
+sudo systemctl start daq.service
diff --git a/bin/mudacl b/bin/mudacl
@@ -13,7 +13,7 @@ port_acls=inst/port_acls
 cd $ROOT
 mudacl/bin/test.sh
 
-source misc/config_base.sh
+source etc/config_base.sh
 
 if [ $# -gt 0 ]; then
     echo Unexpected command line argument: $*

diff --git a/bin/physical_sec b/bin/physical_sec
@@ -7,7 +7,7 @@ fi
 
 ROOT=$(realpath $(dirname $0)/..)
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 iprefix=ctrl
 ctrl_bridge=${iprefix}-br

diff --git a/bin/registrar b/bin/registrar
@@ -3,7 +3,7 @@
 ROOT=$(realpath $(dirname $0)/..)
 cd $ROOT
 
-source misc/config_base.sh
+source etc/config_base.sh
 
 if [ -z "$gcp_cred" ]; then
     echo Need to define [gcp_cred] config variable.

diff --git a/bin/setup_stack b/bin/setup_stack
@@ -140,7 +140,7 @@ add_iface corp 1 faux-3
 echo Starting faucet instances...
 sudo rm -rf inst/faucet
 mkdir -p inst/faucet/daq-faucet-corp
-cp misc/corp_$mode.yaml inst/faucet/daq-faucet-corp/faucet.yaml
+cp topology/corp-$mode/faucet.yaml inst/faucet/daq-faucet-corp/faucet.yaml
 docker kill daq-faucet-corp || true
 sudo modprobe bonding
 cmd/faucet corp 6683

diff --git a/bin/setup_switch b/bin/setup_switch
@@ -45,7 +45,7 @@ EOM
 } 
 interface_setup
 
-curl -u manager:friend "https://$switch_ip/fs/flash/af.cfg" -k -X PUT -H 'Content-Type: ' --data-binary @misc/switch_configs/at.cfg
+curl -u manager:friend "https://$switch_ip/fs/flash/af.cfg" -k -X PUT -H 'Content-Type: ' --data-binary @etc/switch_configs/at.cfg
 
 curl -u manager:friend "https://$switch_ip/api/boot/config" -k -H 'content-type: application/json' --data '{"name":"flash:/af.cfg"}'
 

diff --git a/bin/simple_bridge b/bin/simple_bridge
@@ -7,7 +7,7 @@ iface_b=enx00e04c680029
 rm -rf inst/faucet/daq-faucet-simple
 mkdir -p inst/faucet/daq-faucet-simple
 
-cp misc/corp_bond.yaml inst/faucet/daq-faucet-simple/faucet.yaml
+cp topology/corp-bond/faucet.yaml inst/faucet/daq-faucet-simple/faucet.yaml
 
 cmd/faucet simple 6789
 

diff --git a/bin/techsupport b/bin/techsupport
@@ -37,7 +37,7 @@ done
 CONF_PROBE=inst/config/system.conf
 touch $CONF_PROBE || sudo chown $USER -R inst
 
-source misc/config_base.sh
+source etc/config_base.sh
 ext_ctrl=$switch_setup_ctrl_intf
 ext_addr=$switch_setup_ip_addr
 ext_intf=$switch_setup_data_intf

diff --git a/bin/test_all b/bin/test_all
@@ -2,7 +2,7 @@
 
 # Just sanity checks to make sure it's a proper directory structure.
 test -d bin
-test -d misc
+test -d resources
 test -d cmd
 
 sudo rm -rf inst/ local/ out/

diff --git a/bin/user_enable b/bin/user_enable
@@ -3,6 +3,6 @@
 ROOT=$(dirname $0)/..
 cd $ROOT
 
-source misc/config_base.sh 
+source etc/config_base.sh
 
 python daq/gcp.py register_offenders=true $conf_file 
diff --git a/bin/validate b/bin/validate
@@ -3,7 +3,7 @@
 ROOT=$(realpath $(dirname $0)/..)
 cd $ROOT
 
-source misc/config_base.sh
+source etc/config_base.sh
 
 if [ -z "$gcp_cred" ]; then
   echo Please make sure gcp_cred is defined in local/system.conf

diff --git a/cmd/auto b/cmd/auto
@@ -3,7 +3,7 @@
 ROOT=$(realpath $(dirname $0)/..)
 cd $ROOT
 run_args="$@"
-source misc/config_base.sh
+source etc/config_base.sh
 
 cleanup_file=inst/exrun_cleanup.sh
 CFILE=inst/config/system.conf

diff --git a/cmd/build b/cmd/build
@@ -25,7 +25,7 @@ DOCKER_IMAGE_LIST=docker_images.txt
 DOCKER_IMAGE_VER=docker_images.ver
 
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 host_tests=$host_tests bin/docker_build_files
 function pull_images {
     TAG=$1

diff --git a/cmd/clean b/cmd/clean
@@ -5,7 +5,7 @@ cd $ROOT
 
 # Sanity checks to make sure we're in a proper directory before we go nuke-happy.
 test -d daq
-test -d misc
+test -d resources
 
 sudo rm -rf inst build
 sudo rm -f .build_hash

diff --git a/cmd/exrun b/cmd/exrun
@@ -33,7 +33,7 @@ if ! git show $TAGGED_VERSION > /dev/null; then
 fi
 
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 while [ -f $INSTLOCK ]; do
     echo Waiting for $INSTLOCK to disappear!

diff --git a/cmd/faucet b/cmd/faucet
@@ -14,7 +14,7 @@ docker_volumes=
 port_opts=
 
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 if [ -n "$switch_setup_lo_port" ]; then
     faucet_pt=$switch_setup_lo_port

diff --git a/cmd/faux b/cmd/faux
@@ -13,7 +13,7 @@ image=daqf/faux1
 build=
 
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 # If the argument is -n then create a networking container
 if [ "$1" == "-n" ]; then

diff --git a/cmd/run b/cmd/run
@@ -8,7 +8,7 @@ run_args="$@"
 cmdrun_log=inst/cmdrun.log
 
 cd $ROOT
-source misc/config_base.sh
+source etc/config_base.sh
 
 echo Starting `date`
 echo Clearing previous state...

diff --git a/misc/all_tests.conf → config/modules/all.conf b/misc/all_tests.conf → config/modules/all.conf
@@ -1,5 +1,5 @@
 # Baseline configuration.
-include misc/host_tests.conf
+include config/modules/host.conf
 
 # All contributed modules.
 include subset/switches/build.conf

diff --git a/misc/hold_tests.conf → config/modules/hold.conf b/misc/hold_tests.conf → config/modules/hold.conf
diff --git a/misc/host_tests.conf → config/modules/host.conf b/misc/host_tests.conf → config/modules/host.conf
diff --git a/misc/local_tests.conf → config/modules/local.conf b/misc/local_tests.conf → config/modules/local.conf
@@ -1,7 +1,7 @@
 # Baseline test file for officially included tests.
 
 # Default built-in tests.
-include misc/host_tests.conf
+include config/modules/host.conf
 
 # Local tests to build.
 build local/docker

diff --git a/misc/pass_only.conf → config/modules/pass.conf b/misc/pass_only.conf → config/modules/pass.conf
diff --git a/misc/ping_only.conf → config/modules/ping.conf b/misc/ping_only.conf → config/modules/ping.conf
diff --git a/misc/topo_tests.conf → config/modules/topo.conf b/misc/topo_tests.conf → config/modules/topo.conf
diff --git a/misc/system_all.conf → config/system/all.conf b/misc/system_all.conf → config/system/all.conf
@@ -1,7 +1,7 @@
 # Base configuration file for DAQ.
 
 # Load defaults.
-source misc/system.conf
+source config/system/system.conf
 
 # Description description for dashboard.
 site_description="Multi-Device All-Tests Configuration"
@@ -16,4 +16,4 @@ interfaces.faux-2.opts=
 interfaces.faux-3.opts=
 
 # For this run all tests.
-host_tests=misc/all_tests.conf
+host_tests=config/modules/all.conf
diff --git a/misc/system_base.yaml → config/system/base.yaml b/misc/system_base.yaml → config/system/base.yaml
@@ -5,7 +5,7 @@
 # adapter, modify the interfaces entry with the appripriate interface name(s).
 
 # Load defaults.
-include: misc/system.yaml
+include: config/system/system.yaml
 
 # Description for dashboard.
 site_description: "Baseline Configuration"

diff --git a/misc/system_ext.conf → config/system/ext.conf b/misc/system_ext.conf → config/system/ext.conf
@@ -1,7 +1,7 @@
 # Base configuration file for using an OVS switch external to DAQ itself.
 
 # Load defaults.
-source misc/system.conf
+source config/system/system.conf
 
 # Description description for dashboard.
 site_description="External (not integrated with DAQ) OVS switch configuration"

diff --git a/misc/system_muddy.conf → config/system/muddy.conf b/misc/system_muddy.conf → config/system/muddy.conf
@@ -1,7 +1,7 @@
 # Config file for testing basic MUD configurations.
 
 # Load defaults.
-source misc/system.conf
+source config/system/system.conf
 
 # Description description for dashboard.
 site_description="Multi-Device Configuration"
@@ -15,5 +15,5 @@ interfaces.faux-1.opts=bacnet
 interfaces.faux-2.opts=discover
 interfaces.faux-3.opts=discover
 
-host_tests=misc/ping_only.conf
-test_config=misc/runtime_configs/discover
+host_tests=config/modules/ping.conf
+test_config=resources/runtime_configs/discover
diff --git a/misc/system_multi.conf → config/system/multi.conf b/misc/system_multi.conf → config/system/multi.conf
@@ -1,7 +1,7 @@
 # Base configuration file for DAQ.
 
 # Load defaults.
-source misc/system.conf
+source config/system/system.conf
 
 # Description description for dashboard.
 site_description="Multi-Device Configuration"

diff --git a/misc/system.conf → config/system/system.conf b/misc/system.conf → config/system/system.conf
@@ -3,4 +3,4 @@
 #
 # All concent should be in the new file/format.
 #
-source misc/system.yaml
+source config/system/system.yaml
diff --git a/misc/system.yaml → config/system/system.yaml b/misc/system.yaml → config/system/system.yaml
@@ -36,4 +36,4 @@ dhcp_lease_time: 500s
 long_dhcp_response_sec: 105
 
 # finish hook: executed at the end of every test
-finish_hook: misc/dump_network.sh
+finish_hook: bin/dump_network
diff --git a/daq/runner.py b/daq/runner.py
@@ -40,7 +40,7 @@ class owns the main event loop and shards out work to subclasses."""
     _DEFAULT_RETENTION_DAYS = 30
     _MODULE_CONFIG = 'module_config.json'
     _RUNNER_CONFIG_PATH = 'runner/setup'
-    _DEFAULT_TESTS_FILE = 'misc/host_tests.conf'
+    _DEFAULT_TESTS_FILE = 'config/modules/host.conf'
     _RESULT_LOG_FILE = 'inst/result.log'
 
     def __init__(self, config):