Added EKS tools and reorganized folders (armandmcqueen#58)

.gitignore

@@ -6,7 +6,7 @@ tensorflow-1.13.0-cp36-cp36m-linux_x86_64.whl
 
 
 # EKS configs
-eks/maskrcnn/values/
+infra/eks/maskrcnn/values/
 
 
 # tensorpack-specific stuff

docker/build.sh → infra/docker/build.sh

@@ -8,4 +8,4 @@ echo ""
 
 
 
-docker build -t tensorpack-mask-rcnn:dev-${BRANCH_NAME} .. --build-arg CACHEBUST=$(date +%s) --build-arg BRANCH_NAME=${BRANCH_NAME}
+docker build -t tensorpack-mask-rcnn:dev-${BRANCH_NAME} ../.. --build-arg CACHEBUST=$(date +%s) --build-arg BRANCH_NAME=${BRANCH_NAME}

infra/eks/TOOLS.md

@@ -0,0 +1,27 @@
+# Tools
+
+# Invoke Tasks
+
+We use [invoke](https://www.pyinvoke.org/) to automate tasks around EKS training.
+
+invoke tasks are essentially python functions that can be launched with `inv task_name args` or `invoke task_name args`
+
+## repeat
+
+Repeatedly run a string as command, replacing the substring '|N|' (can be changed) with the iteration number
+
+```
+$ inv repeat 'echo |N|'
+1
+2
+3
+4
+5
+$ inv repeat 'echo [I]' --repeat=2 --verbose --sub='[I]'
+[cmd = echo 1]
+1
+[cmd = echo 2]
+2
+```
+
+## 

eks/eksctl/p3_config.yaml → infra/eks/eksctl/p3_config.yaml

@@ -23,7 +23,7 @@ vpc:
         id: "subnet-21ac2f2e"
 
 nodeGroups:
-  - name: ng-p3
+  - name: ng-p3-1f
     instanceType: p3.16xlarge
     availabilityZones: ["us-east-1f"]
     desiredCapacity: 1

eks/eksctl/p3dn_additional_nodegroup_non_gpu.yaml → infra/eks/eksctl/p3dn_additional_nodegroup_non_gpu.yaml

@@ -3,11 +3,11 @@ apiVersion: eksctl.io/v1alpha5
 kind: ClusterConfig
 
 metadata:
-  name: armand-tensorpack-mask-rcnn
+  name: armand-tensorpack-mask-rcnn-p3dn
   region: us-east-1
 
 nodeGroups:
-  - name: ng-c5
+  - name: ng-c5-1f
     instanceType: c5.2xlarge
     availabilityZones: ["us-east-1f"]
     desiredCapacity: 1

eks/eksctl/p3dn_config.yaml → infra/eks/eksctl/p3dn_config.yaml

@@ -4,7 +4,7 @@ apiVersion: eksctl.io/v1alpha5
 kind: ClusterConfig
 
 metadata:
-  name: armand-tensorpack-mask-rcnn
+  name: armand-tensorpack-mask-rcnn-p3dn
   region: us-east-1
 
 
@@ -23,7 +23,7 @@ vpc:
         id: "subnet-21ac2f2e"
 
 nodeGroups:
-  - name: ng-p3dn
+  - name: ng-p3dn-1c
     instanceType: p3dn.24xlarge
     availabilityZones: ["us-east-1c"]
     desiredCapacity: 1

eks/eksctl/p3dn_create.sh → infra/eks/eksctl/p3dn_create.sh

@@ -4,7 +4,7 @@
 
 eksctl create cluster -f p3dn_config.yaml --auto-kubeconfig
 
-export KUBECONFIG=/Users/armanmcq/.kube/eksctl/clusters/armand-tensorpack-mask-rcnn
+export KUBECONFIG=/Users/armanmcq/.kube/eksctl/clusters/armand-tensorpack-mask-rcnn-p3dn
 # aws eks --region $AWS_REGION update-kubeconfig --name $EKS_CLUSTER
 
 kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/1.0.0-beta/nvidia-device-plugin.yml

eks/eksctl/p3dn_kubeconfig → infra/eks/eksctl/p3dn_kubeconfig

@@ -1 +1 @@
-export KUBECONFIG=/Users/armanmcq/.kube/eksctl/clusters/armand-tensorpack-mask-rcnn
+export KUBECONFIG=/Users/armanmcq/.kube/eksctl/clusters/armand-tensorpack-mask-rcnn-p3dn

infra/eks/maskrcnn/overlays/run10.yaml

@@ -0,0 +1,3 @@
+append:
+  global:
+    name: -run10

infra/eks/maskrcnn/overlays/run6.yaml

@@ -0,0 +1,3 @@
+append:
+  global:
+    name: -run6

infra/eks/maskrcnn/overlays/run7.yaml

@@ -0,0 +1,3 @@
+append:
+  global:
+    name: -run7

infra/eks/maskrcnn/overlays/run8.yaml

@@ -0,0 +1,3 @@
+append:
+  global:
+    name: -run8

infra/eks/maskrcnn/overlays/run9.yaml

@@ -0,0 +1,3 @@
+append:
+  global:
+    name: -run9

infra/eks/parse_logs

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import json
+
+from typing import List, Dict
+import argparse
+
+
+def extract_results_from_log(log_lines: List[str]) -> Dict[str, float]:
+
+    result_lines = {}
+
+    for line in log_lines:
+        if 'Total duration' in line:
+            result_lines['duration'] = line
+
+        elif "mAP(bbox)/IoU=0.5: " in line:
+            result_lines['bbox-0.5'] = line
+
+        elif "mAP(bbox)/IoU=0.5:0.95: " in line:
+            result_lines['bbox-0.5:0.95'] = line
+
+        elif "mAP(bbox)/IoU=0.75: " in line:
+            result_lines['bbox-0.75'] = line
+
+        elif "mAP(bbox)/large: " in line:
+            result_lines['bbox-large'] = line
+
+        elif "mAP(bbox)/medium: " in line:
+            result_lines['bbox-medium'] = line
+
+        elif "mAP(bbox)/small: " in line:
+            result_lines['bbox-small'] = line
+
+        elif "mAP(segm)/IoU=0.5: " in line:
+            result_lines['segm-0.5'] = line
+
+        elif "mAP(segm)/IoU=0.5:0.95: " in line:
+            result_lines['segm-0.5:0.95'] = line
+
+        elif "mAP(segm)/IoU=0.75: " in line:
+            result_lines['segm-0.75'] = line
+
+        elif "mAP(segm)/large: " in line:
+            result_lines['segm-large'] = line
+
+        elif "mAP(segm)/medium: " in line:
+            result_lines['segm-medium'] = line
+
+        elif "mAP(segm)/small: " in line:
+            result_lines['segm-small'] = line
+
+        else:
+            continue
+
+    # Extract result from end fof line as a string
+    def result(result_line):
+        return float(result_line.split(": ")[1].strip().replace(",", ""))
+
+
+    return {
+        "bbox-0.5":      result(result_lines['bbox-0.5']),
+        "bbox-0.5:0.95": result(result_lines['bbox-0.5:0.95']),
+        'bbox-0.75':     result(result_lines['bbox-0.75']),
+        'bbox-small':    result(result_lines['bbox-small']),
+        'bbox-medium':   result(result_lines['bbox-medium']),
+        'bbox-large':    result(result_lines['bbox-large']),
+
+        'segm-0.5':      result(result_lines['segm-0.5']),
+        'segm-0.5:0.95': result(result_lines['segm-0.5:0.95']),
+        'segm-0.75':     result(result_lines['segm-0.75']),
+        'segm-small':    result(result_lines['segm-small']),
+        'segm-medium':   result(result_lines['segm-medium']),
+        'segm-large':    result(result_lines['segm-large']),
+
+        'duration':      result(result_lines['duration'])
+    }
+
+
+
+def order_results(parsed_results, ordered_fields):
+    return
+
+
+if __name__ == '__main__':
+    ordered_fields = [
+        'bbox-0.5',
+        'bbox-0.5:0.95',
+        'bbox-0.75',
+        'bbox-large',
+        'bbox-medium',
+        'bbox-small',
+        'segm-0.5',
+        'segm-0.5:0.95',
+        'segm-0.75',
+        'segm-large',
+        'segm-medium',
+        'segm-small',
+        'duration',
+    ]
+    parser = argparse.ArgumentParser(
+        description='Parse results from maskrcnn train logs')
+    parser.add_argument('log_path',
+                        help="Yaml to use as base. "
+                             "If '-' is given, will read from stdin instead."
+                             "If 'csv-header', will print csv header line")
+    parser.add_argument('--output_type', type=str, choices=["csv", "json", "pretty-json", "kv", "yaml"],
+                        help='Output format.')
+
+    args = parser.parse_args()
+
+    if args.log_path == "csv-header":
+        sys.stdout.write(",".join(ordered_fields))
+        sys.stdout.write("\n")
+        quit()
+
+    if args.log_path == '-':
+        log_lines = [l for l in sys.stdin]
+    else:
+        with open(args.log_path, 'r') as f:
+            log_lines = [line for line in f]
+
+    results = extract_results_from_log(log_lines)
+
+    if args.output_type == "csv":
+        sys.stdout.write(",".join([str(results[f]) for f in ordered_fields]))
+    elif args.output_type == "json":
+        sys.stdout.write(json.dumps(results))
+    elif args.output_type == "pretty-json":
+        sys.stdout.write(json.dumps(results, indent=4))
+    elif args.output_type == "kv" or args.output_type == "yaml":
+        if args.output_type == "yaml":
+            link = ": "
+        else:
+            link = "="
+        for i, field in enumerate(ordered_fields):
+            result = results[field]
+            if args.output_type == "yaml":
+                field = field.replace("-", "_")
+
+            kv = "\n" if i > 0 else ""
+            kv += field + link + str(result)
+            sys.stdout.write(kv)
+    else:
+        raise RuntimeError("Unrecognized choice "+args.output_type+". This should not occur")
+
+    sys.stdout.write("\n")