adjustment according to asplos-ae suggestions

Author: lzjzx1122

README.md

@@ -10,13 +10,17 @@ FaaSFlow is a serverless workflow engine that enables efficient workflow executi
 
 2. Please save the private IP address of the storage node as the **<master_ip>**, and save the private IP address of the other 7 worker nodes as the **<worker_ip>**. 
 
+## About Config Setting
+
+There are 2 places for config setting. `src/container/container_config.py` specifies CouchDB and Redis's address, you need to fill in correct ip so that application code can directly connect to database inside container environment. All other configurations are in `config/config.py`.
+
 ## Installation and Software Dependencies
 
 Clone our code `https://github.com/lzjzx1122/FaaSFlow.git` and:
 
 1. Reset `worker_address` configuration with your <worker_ip>:8000 on `src/grouping/node_info.yaml`. It will specify your workers' addresses. The `scale_limit: 120` represents the maximum container numbers that can be deployed in each 32GB memory instance, and it does not need any change by default.
 
-2. Reset `COUCHDB_URL` as `http://openwhisk:openwhisk@<master_ip>:5984/`  in `src/container/config.py`, `src/workflow_manager/config.py`, `test/asplos/config.py`. It will specify the corresponding database storage you built previously.
+2. Reset `COUCHDB_URL` as `http://openwhisk:openwhisk@<master_ip>:5984/`  in `config/config.py`, `src/container/container_config.py`. It will specify the corresponding database storage you built previously.
 
 3. Then, clone the modified code into each node (8 nodes total).
 
@@ -30,11 +34,13 @@ Clone our code `https://github.com/lzjzx1122/FaaSFlow.git` and:
 
 ## WorkerSP Start-up
 
-The following operations help to run scripts under WorkerSP. Firstly, enter `src/workflow_manager`, change the configuration by `DATA_MODE = optimized` and `CONTROL_MODE = WorkerSP` in both 7 worker nodes and storage node. Then, start the engine proxy with the local  <worker_ip> on each worker node by the following <span id="jump">command</span>: 
+The following operations help to run scripts under WorkerSP.
+
+Firstly, change the configuration by `DATA_MODE = optimized` and `CONTROL_MODE = WorkerSP` in both 7 worker nodes and storage node. Define the `GATEWAY_ADDR` as `<master_ip>:7000`. Then, enter `src/workflow_manager` and start the engine proxy with the local  <worker_ip> on each worker node by the following <span id="jump">command</span>: 
 ```
     python3 proxy.py <worker_ip> 8000             (proxy start)
 ```
-Enter `test/asplos/config.py` and define the `GATEWAY_ADDR` as `<master_ip>:7000`. Then start the gateway on the storage node by the following command: 
+Then start the gateway on the storage node by the following command: 
 ```
     python3 gateway.py <master_ip> 7000           (gateway start)
 ``` 
@@ -44,9 +50,9 @@ If you would like to run scripts under WorkerSP, you have finished all the opera
 
 ## MasterSP Start-up
 
-The following operations help to run scripts under MasterSP. Firstly, enter `src/workflow_manager`, change the configuration by `DATA_MODE = raw` and `CONTROL_MODE = MasterSP` in both 7 worker nodes and storage node. Then, restart the engine proxy on each worker node by the [proxy start](#jump) command, and restart the gateway on the storage node by the [gateway start](#jump) command.
+The following operations help to run scripts under MasterSP. Firstly, change the configuration by `DATA_MODE = raw` and `CONTROL_MODE = MasterSP` in both 7 worker nodes and storage node. Then, restart the engine proxy on each worker node by the [proxy start](#jump) command, and restart the gateway on the storage node by the [gateway start](#jump) command.
 
-Enter `src/workflow_manager/config.py`, and define the `MASTER_HOST` as `<master_ip>:8000`. Then,
+Define the `MASTER_HOST` as `<master_ip>:8000`. Then,
 start another proxy on the storage node as the virtual master node by the following command: 
 ```
     python3 proxy.py <master_ip> 8000

benchmark/fileprocessing/sentiment/main.py

@@ -1,11 +1,10 @@
 import nltk
 from nltk.sentiment import SentimentIntensityAnalyzer
-import time
-# nltk.download('vader_lexicon')
+nltk.download('vader_lexicon')
 
 def main():
     content = store.fetch(['file'])['file']
     sia = SentimentIntensityAnalyzer()
-    score = sia.polarity_scores(content) # can't handle large dataset
-    # time.sleep(7)
+    score = sia.polarity_scores(content)
+    # time.sleep(7) # if can't handle large dataset, please uncomment this to simulate sentiment processing
     store.put({'score': 0}, {})

benchmark/generator/translator.py

@@ -1,8 +1,5 @@
 import json
-import yaml
-import sys
 import os
-import random
 
 def translator(workflow_name: str):
     ## parse json build flat_workflow.yaml

benchmark/illgal_recognizer/extract/Dockerfile

@@ -1,6 +1,5 @@
 FROM workflow_base
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
 RUN apt-get update
 RUN apt-get install -y tesseract-ocr libgl1-mesa-glx

benchmark/illgal_recognizer/mosaic/Dockerfile

@@ -1,6 +1,5 @@
 FROM workflow_base
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
 RUN apt-get update
 RUN apt-get install -y tesseract-ocr libgl1-mesa-glx

benchmark/video/merge/Dockerfile

@@ -1,9 +1,6 @@
 FROM workflow_base
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
-RUN sed -i 's/security.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
-
 RUN apt-get update
 RUN apt-get -y --force-yes --fix-missing install yasm ffmpeg
 

benchmark/video/simple_process/Dockerfile

@@ -1,9 +1,6 @@
 FROM workflow_base
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
-RUN sed -i 's/security.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
-
 RUN apt-get update
 RUN apt-get -y --force-yes install yasm ffmpeg
 

benchmark/video/split/Dockerfile

@@ -1,9 +1,6 @@
 FROM workflow_base
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
-RUN sed -i 's/security.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
-
 RUN apt-get update
 RUN apt-get -y --force-yes install yasm ffmpeg
 

benchmark/video/transcode/Dockerfile

@@ -1,9 +1,6 @@
 FROM workflow_base
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
-RUN sed -i 's/security.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
-
 RUN apt-get update
 RUN apt-get -y --force-yes install yasm ffmpeg
 

benchmark/video/upload/Dockerfile

@@ -1,9 +1,6 @@
 FROM workflow_base
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
-RUN sed -i 's/security.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
-
 RUN apt-get update
 RUN apt-get -y --force-yes install yasm ffmpeg
 

config/config.py

@@ -0,0 +1,29 @@
+COUCHDB_URL = 'http://openwhisk:[email protected]:5984/'
+REDIS_HOST = '10.2.64.8'
+REDIS_PORT = 6380
+REDIS_DB = 0
+GATEWAY_ADDR = '10.2.64.8:7000'
+MASTER_HOST = '10.2.64.8:8000'
+WORKFLOW_YAML_ADDR = {'fileprocessing': '/home/openwhisk/Workflow/benchmark/fileprocessing/flat_workflow.yaml',
+                  'illgal_recognizer': '/home/openwhisk/Workflow/benchmark/illgal_recognizer/flat_workflow.yaml',
+                  'video': '/home/openwhisk/Workflow/benchmark/video/flat_workflow.yaml',
+                  'wordcount': '/home/openwhisk/Workflow/benchmark/wordcount/flat_workflow.yaml',
+                  'cycles': '/home/openwhisk/Workflow/benchmark/generator/cycles/flat_workflow.yaml',
+                  'epigenomics': '/home/openwhisk/Workflow/benchmark/generator/epigenomics/flat_workflow.yaml',
+                  'genome': '/home/openwhisk/Workflow/benchmark/generator/genome/flat_workflow.yaml',
+                  'soykb': '/home/openwhisk/Workflow/benchmark/generator/soykb/flat_workflow.yaml'}
+NETWORK_BANDWIDTH = 25 * 1024 * 1024 / 4 # 25MB/s / 4
+NET_MEM_BANDWIDTH_RATIO = 15 # mem_time = net_time / 15
+CONTAINER_MEM = 256 * 1024 * 1024 # 256MB
+NODE_MEM = 256 * 1024 * 1024 * 1024 # 256G
+RESERVED_MEM_PERCENTAGE = 0.2
+GROUP_LIMIT = 100
+RPMs = {'genome-25': [2, 4, 6, 8], 'genome-50': [2, 4, 6, 8, 10], 'genome-75': [2, 4, 6, 8, 10], 'genome-100': [2, 4, 6, 8, 10],
+'video-25': [4, 8, 16, 24], 'video-50': [8, 16, 24, 32, 40], 'video-75': [8, 16, 24, 32, 40], 'video-100': [8, 16, 24, 32, 40]}
+FUNCTION_INFO_ADDRS = {'genome': '../../benchmark/generator/genome', 'epigenomics': '../../benchmark/generator/epigenomics',
+                                                'soykb': '../../benchmark/generator/soykb', 'cycles': '../../benchmark/generator/cycles',
+                                                'fileprocessing': '../../benchmark/fileprocessing', 'wordcount': '../../benchmark/wordcount',
+                                                'illgal_recognizer': '../../benchmark/illgal_recognizer', 'video': '../../benchmark/video'}
+DATA_MODE = 'raw' # raw, optimized
+CONTROL_MODE = 'WorkerSP' # WorkerSP, MasterSP
+CLEAR_DB_AND_MEM = True

scripts/requirements.txt

@@ -4,4 +4,5 @@ couchdb
 docker
 flask
 psutil
-pandas
+pandas
+PyYAML

src/container/Dockerfile

@@ -12,7 +12,7 @@ RUN mkdir /proxy
 COPY proxy.py /proxy/proxy.py
 COPY main.py /proxy/main.py
 COPY Store.py /proxy/Store.py
-COPY config.py /proxy/config.py
+COPY container_config.py /proxy/container_config.py
 
 WORKDIR /proxy
 
@@ -25,7 +25,6 @@ EXPOSE 5000
 #     pip install --no-cache-dir gevent flask && \
 #     apk del .build-deps
 
-RUN sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list
 RUN apt-get clean
 RUN apt-get update
 RUN pip3 install --no-cache-dir gevent flask couchdb redis

src/container/config.py

@@ -0,0 +1,4 @@
+COUCHDB_URL = 'http://openwhisk:[email protected]:5984/'
+REDIS_HOST = '172.17.0.1'
+REDIS_PORT = 6380
+REDIS_DB = 0

src/container/container_config.py

@@ -4,14 +4,14 @@
 from flask import Flask, request
 from gevent.pywsgi import WSGIServer
 from Store import Store
-import config
+import container_config
 import redis
 
 default_file = 'main.py'
 work_dir = '/proxy'
-couchdb_url = config.COUCHDB_URL
+couchdb_url = container_config.COUCHDB_URL
 db_server = couchdb.Server(couchdb_url)
-redis_server = redis.StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB)
+redis_server = redis.StrictRedis(host=container_config.REDIS_HOST, port=container_config.REDIS_PORT, db=container_config.REDIS_DB)
 latency_db = db_server['workflow_latency']
 
 class Runner:

src/container/proxy.py

@@ -1,14 +1,12 @@
 import sys
-from typing import Dict, List
 import parse_yaml
 import queue
-import json
 import component
 import repository
-import config
-import time
 import yaml
-import uuid
+
+sys.path.append('../../config')
+import config
 
 mem_usage = 0
 max_mem_usage = 0

src/grouping/config.py

@@ -1,15 +1,15 @@
 nodes:
-- worker_address: 172.20.185.151:8000
-  scale_limit: 120
-- worker_address: 172.20.185.152:8000
-  scale_limit: 120
-- worker_address: 172.20.185.153:8000
-  scale_limit: 120
-- worker_address: 172.20.185.154:8000
-  scale_limit: 120
-- worker_address: 172.20.185.155:8000
-  scale_limit: 120
-- worker_address: 172.20.185.156:8000
-  scale_limit: 120
-- worker_address: 172.20.185.157:8000
-  scale_limit: 120
+- worker_address: 10.2.64.8:8000
+  scale_limit: 120
+# - worker_address: 172.20.185.152:8000
+#   scale_limit: 120
+# - worker_address: 172.20.185.153:8000
+#   scale_limit: 120
+# - worker_address: 172.20.185.154:8000
+#   scale_limit: 120
+# - worker_address: 172.20.185.155:8000
+#   scale_limit: 120
+# - worker_address: 172.20.185.156:8000
+#   scale_limit: 120
+# - worker_address: 172.20.185.157:8000
+#   scale_limit: 120

src/grouping/grouping.py

@@ -1,5 +1,8 @@
 import yaml
 import component
+import sys
+
+sys.path.append('../../config')
 import config
 
 yaml_file_addr = config.WORKFLOW_YAML_ADDR

src/grouping/node_info.yaml

@@ -1,7 +1,9 @@
-from os import remove
 import couchdb
 import redis
 from typing import Dict, List
+import sys
+
+sys.path.append('../../config')
 import config
 
 couchdb_url = config.COUCHDB_URL

src/grouping/parse_yaml.py

@@ -7,6 +7,8 @@
 from repository import Repository
 import requests
 import time
+
+sys.path.append('../../config')
 import config
 
 app = Flask(__name__)

src/grouping/repository.py

@@ -4,14 +4,12 @@
 import gevent
 import json
 from typing import Dict
-from threading import Thread
+import sys
+sys.path.append('../../config')
 import config
 from workersp import WorkerSPManager
 from mastersp import MasterSPManager
 import docker
-import subprocess
-
-import sys
 from flask import Flask, request
 app = Flask(__name__)
 docker_client = docker.from_env()

src/workflow_manager/config.py

@@ -2,6 +2,9 @@
 import couchdb
 import redis
 import json
+import sys
+
+sys.path.append('../../config')
 import config
 
 couchdb_url = config.COUCHDB_URL

src/workflow_manager/gateway.py

@@ -5,6 +5,7 @@
 import getopt
 import sys
 sys.path.append('..')
+sys.path.append('../../../config')
 from repository import Repository
 import config
 import pandas as pd

src/workflow_manager/proxy.py

@@ -5,6 +5,7 @@
 import getopt
 import sys
 sys.path.append('..')
+sys.path.append('../../../config')
 from repository import Repository
 import config
 import pandas as pd

src/workflow_manager/repository.py

@@ -5,6 +5,7 @@
 import getopt
 import sys
 sys.path.append('..')
+sys.path.append('../../../config')
 from repository import Repository
 import config
 import pandas as pd