vhive-serverless · plamenmpetrov · Dec 11, 2020 · Dec 11, 2020 · Dec 13, 2020 · Dec 13, 2020
diff --git a/ae_scripts/functions.txt b/ae_scripts/functions.txt
@@ -0,0 +1,10 @@
+helloworld
+chameleon
+pyaes
+image_rotate_s3
+json_serdes_s3
+lr_serving
+cnn_serving
+rnn_serving
+lr_training_s3
+video_processing_s3
diff --git a/ae_scripts/run_all.sh b/ae_scripts/run_all.sh
@@ -0,0 +1,142 @@
+# MIT License
+#
+# Copyright (c) 2020 Dmitrii Ustiugov, Plamen Petrov and EASE lab
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+#!/bin/bash
+
+set -e
+
+# Kill all children of the process upon a keyboard interrupt or exit
+trap "exit" INT TERM
+trap "kill 0" EXIT
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+ROOT="$( cd $DIR && cd .. && pwd)"
+
+source /etc/profile
+GO_BIN=`which go`
+
+cd $ROOT
+
+############## Supplementary functions below ##############
+die () {
+    echo >&2 "$@"
+    exit 1
+}
+
+################### Main body below #######################
+
+[ "$#" -eq 1 ] || die "1 argument required, $# provided"
+
+mode=$1
+
+if [[ ! $mode =~ ^(baseline|reap)$ ]]; then
+    die "Wrong mode specified, the scripts supports only the following modes: baseline, reap"
+fi
+
+host_ip=`curl ifconfig.me`
+
+# Make sure KVM is available on the machine
+sudo setfacl -m u:${USER}:rw /dev/kvm
+
+# Reading the file with functions
+i=0
+for j in `cat $ROOT/ae_scripts/functions.txt`
+do
+    wlds[$i]=$j
+    i=$(($i+1))
+done
+
+echo The experiment will run the following functions: ${wlds[@]}
+
+all_results_path=all_results
+results_path=$all_results_path/$mode
+rm -rf $results_path || echo Folder $results_path exists, removing the old one.
+mkdir -p $results_path
+
+echo Run MinIO server as a daemon
+sudo pkill -9 minio || echo
+$ROOT/function-images/minio_scripts/start_minio_server.sh 1>/dev/null &
+sleep 1
+
+echo ======================================================
+echo ============== Starting the experiment ===============
+echo ======================================================
+
+for wld in "${wlds[@]}"
+do
+    echo
+    echo About to run $mode/$wld experiment in $mode mode
+
+    if [[ "$mode" == "reap" ]]; then
+        modeFlag=-upfTest
+    fi
+
+    ############ Clean up after previous experiment ########
+    echo Killing the containerd daemon and cleaning up.
+    ./scripts/clean_fcctr.sh 1>clean.out 2>clean.err
+    wld_dir=$results_path/$wld
+    mkdir -p $wld_dir
+
+    ########################################################
+    echo Starting containerd daemon
+    sudo PATH=$PATH /usr/local/bin/firecracker-containerd \
+        --config /etc/firecracker-containerd/config.toml \
+        1>$wld_dir/containerd.out 2>$wld_dir/containerd.err &
+
+    echo Wait for containerd to start
+    sleep 2
+
+    ########################################################
+    echo Running the actual benchmark... may take up to one minute.
+
+    sudo $GO_BIN test -v -run TestBenchServe \
+        -args -iter 5 \
+        -snapshotsTest \
+        -benchDirTest $results_path/$wld \
+        -metricsTest \
+        -funcName $wld \
+        -minioAddress $host_ip:9000 \
+        $modeFlag 1>$wld_dir/test.out 2>$wld_dir/test.err
+
+    sleep 1
+
+    ################# Process latency stats #################
+    addInstanceMetric=`cat $wld_dir/serve.csv | tail -n 1 | cut -d"," -f2`
+    if [[ "$mode" == "baseline" ]]; then
+        funcInvocationMetric=`cat $wld_dir/serve.csv | tail -n 1 | cut -d"," -f8`
+    else
+        funcInvocationMetric=`cat $wld_dir/serve.csv | tail -n 1 | cut -d"," -f10`
+    fi
+
+    totalMetric=$(( (addInstanceMetric + funcInvocationMetric)/1000 ))
+
+    echo Average function invocation took $totalMetric milliseconds
+
+    echo $wld,$mode,$totalMetric >> $all_results_path/results.csv
+done
+
+sudo pkill -9 minio
+
+echo ======================================================
+echo ====================== All done! =====================
+echo ======================================================
diff --git a/ae_scripts/setup.sh b/ae_scripts/setup.sh
@@ -0,0 +1,72 @@
+# MIT License
+#
+# Copyright (c) 2020 Dmitrii Ustiugov, Plamen Petrov and EASE lab
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+#!/bin/bash
+
+# Kill all children of the process upon a keyboard interrupt or exit
+trap "exit" INT TERM ERR
+trap "kill 0" EXIT
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+ROOT="$( cd $DIR && cd .. && pwd)"
+SCRIPTS=$ROOT/scripts
+
+source $SCRIPTS/install_go.sh
+$SCRIPTS/setup_system.sh
+
+sudo apt-get -y install gcc g++ acl gcc g++ make acl net-tools
+
+# Set up KVM
+sudo setfacl -m u:${USER}:rw /dev/kvm
+
+# Check if KVM is available
+[ -r /dev/kvm ] && [ -w /dev/kvm ] && echo "KVM is available" || echo "KVM is unavailable"
+
+$SCRIPTS/setup_containerd.sh
+
+$SCRIPTS/create_devmapper.sh
+
+echo Set up MinIO server
+$ROOT/function-images/minio_scripts/install_minio.sh
+
+echo Run MinIO server as a daemon
+$ROOT/function-images/minio_scripts/start_minio_server.sh &
+sleep 1
+
+host_ip=`curl ifconfig.me`
+$ROOT/function-images/minio_scripts/create_minio_bucket.sh http://$host_ip:9000 || echo Minio bucket exists, continuing...
+
+echo Populate the bucket with all files
+$ROOT/function-images/minio_scripts/put_in_bucket.sh
+
+echo Contents of the MinIO bucket:
+mc ls myminio/mybucket
+
+sudo pkill -9 minio || echo
+
+echo Build the project
+
+source /etc/profile
+GO_BIN=`which go`
+
+go build
diff --git a/bench_test.go b/bench_test.go
@@ -354,19 +354,19 @@ func getOutFile(name string) string {
 
 func getAllImages() map[string]string {
 	return map[string]string{
-		"helloworld":          "ustiugov/helloworld:var_workload",
-		"chameleon":           "ustiugov/chameleon:var_workload",
-		"pyaes":               "ustiugov/pyaes:var_workload",
-		"image_rotate":        "ustiugov/image_rotate:var_workload",
-		"image_rotate_s3":     "ustiugov/image_rotate_s3:var_workload",
-		"json_serdes":         "ustiugov/json_serdes:var_workload",
-		"json_serdes_s3":      "ustiugov/json_serdes_s3:var_workload",
-		"lr_serving":          "ustiugov/lr_serving:var_workload",
-		"cnn_serving":         "ustiugov/cnn_serving:var_workload",
-		"rnn_serving":         "ustiugov/rnn_serving:var_workload",
-		"lr_training_s3":      "ustiugov/lr_training_s3:var_workload",
-		"lr_training":         "ustiugov/lr_training:var_workload",
-		"video_processing_s3": "ustiugov/video_processing_s3:var_workload",
+		"helloworld":          "vhiveease/helloworld:var_workload",
+		"chameleon":           "vhiveease/chameleon:var_workload",
+		"pyaes":               "vhiveease/pyaes:var_workload",
+		"image_rotate":        "vhiveease/image_rotate:var_workload",
+		"image_rotate_s3":     "vhiveease/image_rotate_s3:var_workload",
+		"json_serdes":         "vhiveease/json_serdes:var_workload",
+		"json_serdes_s3":      "vhiveease/json_serdes_s3:var_workload",
+		"lr_serving":          "vhiveease/lr_serving:var_workload",
+		"cnn_serving":         "vhiveease/cnn_serving:var_workload",
+		"rnn_serving":         "vhiveease/rnn_serving:var_workload",
+		"lr_training_s3":      "vhiveease/lr_training_s3:var_workload",
+		"lr_training":         "vhiveease/lr_training:var_workload",
+		"video_processing_s3": "vhiveease/video_processing_s3:var_workload",
 	}
 }
 

diff --git a/ctriface/go.sum b/ctriface/go.sum
@@ -241,6 +241,7 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -270,6 +271,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
@@ -438,6 +440,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

diff --git a/ctriface/iface.go b/ctriface/iface.go
@@ -115,6 +115,7 @@ func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (_ *
 		containerd.WithSnapshotter(o.snapshotter),
 		containerd.WithNewSnapshot(vmID, *vm.Image),
 		containerd.WithNewSpec(
+			oci.WithEnv([]string{fmt.Sprintf("%s=%s", minioEnvVarKey, o.minioAddress)}),
 			oci.WithImageConfig(*vm.Image),
 			firecrackeroci.WithVMID(vmID),
 			firecrackeroci.WithVMNetwork,

diff --git a/ctriface/orch.go b/ctriface/orch.go
@@ -50,6 +50,7 @@ const (
 	containerdAddress      = "/run/firecracker-containerd/containerd.sock"
 	containerdTTRPCAddress = containerdAddress + ".ttrpc"
 	namespaceName          = "firecracker-containerd"
+	minioEnvVarKey         = "MINIO_ADDRESS"
 )
 
 // Orchestrator Drives all VMs
@@ -65,6 +66,7 @@ type Orchestrator struct {
 	isLazyMode       bool
 	snapshotsDir     string
 	isMetricsMode    bool
+	minioAddress     string
 
 	memoryManager *manager.MemoryManager
 }

diff --git a/ctriface/orch_options.go b/ctriface/orch_options.go
@@ -72,3 +72,10 @@ func WithMetricsMode(isMetricsMode bool) OrchestratorOption {
 		o.isMetricsMode = isMetricsMode
 	}
 }
+
+// WithMinioAddress Sets the IP address and port of the minio server
+func WithMinioAddress(minioAddress string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.minioAddress = minioAddress
+	}
+}
diff --git a/fccd-orchestrator_test.go b/fccd-orchestrator_test.go
@@ -48,6 +48,7 @@ var (
 	isLazyModeTest         = flag.Bool("lazyTest", false, "Enable lazy serving mode when UPFs are enabled")
 	isWithCache            = flag.Bool("withCache", false, "Do not drop the cache before measurements")
 	benchDir               = flag.String("benchDirTest", "bench_results", "Directory where stats should be saved")
+	minioAddress           = flag.String("minioAddress", "", "Address of Minio server holding the inputs to s3 workloads. ADDRESS:PORT")
 )
 
 func TestMain(m *testing.M) {
@@ -79,6 +80,7 @@ func TestMain(m *testing.M) {
 		ctriface.WithUPF(*isUPFEnabledTest),
 		ctriface.WithMetricsMode(*isMetricsModeTest),
 		ctriface.WithLazyMode(*isLazyModeTest),
+		ctriface.WithMinioAddress(*minioAddress),
 	)
 
 	ret := m.Run()

diff --git a/function-images/chameleon/Dockerfile b/function-images/chameleon/Dockerfile
@@ -1,9 +1,9 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN pip3 install --user -r requirements.txt
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ --update --no-cache

diff --git a/function-images/cnn_serving/Dockerfile b/function-images/cnn_serving/Dockerfile
@@ -8,15 +8,17 @@ RUN apk update && \
     ln -sf /usr/bin/pip3 /usr/local/bin/pip && \
     ln -sf /usr/bin/python3 /usr/local/bin/python && \
     ln -sf /usr/bin/python3 /usr/local/bin/python3 && \
-    echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
-    echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+    ln -s /usr/include/locale.h /usr/include/xlocale.h && \
+    echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
+    echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
     apk update && \
     apk add --upgrade && \
     apk add --update --no-cache build-base gcc g++ protobuf && \
     apk add --allow-untrusted --repository http://dl-3.alpinelinux.org/alpine/edge/testing hdf5 hdf5-dev && \
     apk add py3-numpy && \
     apk add py-numpy-dev && \
     pip3 install --upgrade pip && \
+    pip3 uninstall -y enum34 && \
     pip3 install --no-cache-dir Cython && \
     pip3 install --no-cache-dir -r requirements.txt && \
     pip3 install --no-cache-dir protobuf==3.11.3 grpcio==${GRPC_PYTHON_VERSION}

diff --git a/function-images/helloworld/Dockerfile b/function-images/helloworld/Dockerfile
@@ -1,9 +1,9 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN pip3 install --user -r requirements.txt
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ --update --no-cache

diff --git a/function-images/image_rotate_s3/Dockerfile b/function-images/image_rotate_s3/Dockerfile
@@ -1,12 +1,12 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN apk add --no-cache --virtual .build-deps \
     build-base linux-headers zlib-dev jpeg-dev \
     && pip3 install --user -r requirements.txt \
     && apk del .build-deps
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ libjpeg-turbo --update --no-cache