diff --git a/Makefile b/Makefile index 8e4f6f655..1ce4acd6c 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,8 @@ WITHUPF:= WITHLAZY:= WITHSNAPSHOTS:=-snapshotsTest CTRDLOGDIR:=/tmp/ctrd-logs +FIRECRACKER_BIN?=/usr/local/bin/firecracker-containerd +FIRECRACKER_CONFIG_PATH?=/etc/firecracker-containerd/config.toml vhive: proto go install github.com/vhive-serverless/vhive @@ -47,37 +49,109 @@ test-orch: test test-man test: ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log.err & - sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log.err & - sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_upf_log.out 2>$(CTRDLOGDIR)/fccd_orch_upf_log.err & - sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF) - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.out 2>$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.err & - sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY) - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err & - sudo env "PATH=$(PATH)" go test -short $(EXTRAGOARGS) -run TestProfileSingleConfiguration -args -test -loadStep 100 && sudo rm -rf bench_results - sudo env "PATH=$(PATH)" go test -short $(EXTRAGOARGS) -run TestProfileIncrementConfiguration -args -test -vmIncrStep 4 -maxVMNum 4 -loadStep 100 && sudo rm -rf bench_results - sudo env "PATH=$(PATH)" go test -short $(EXTRAGOARGS) -run TestBindSocket - ./scripts/clean_fcctr.sh + sudo mkdir -m777 -p $(CTRDLOGDIR) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log.err" \ + go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log.err" \ + go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_upf_log.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_upf_log.err" \ + go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=/usr/local/bin/firecracker-containerd" \ + "FIRECRACKER_CONFIG_PATH=/etc/firecracker-containerd/config.toml" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.err" \ + go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test -short $(EXTRAGOARGS) -run TestProfileSingleConfiguration \ + -args -test -loadStep 100 && sudo rm -rf bench_results + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test -short $(EXTRAGOARGS) -run TestProfileIncrementConfiguration \ + -args -test -vmIncrStep 4 -maxVMNum 4 -loadStep 100 && sudo rm -rf bench_results + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test -short $(EXTRAGOARGS) -run TestBindSocket test-man: ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_both_log_man_travis.out 2>$(CTRDLOGDIR)/fccd_orch_both_log_man_travis.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) $(WITHUPF) - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_travis.out 2>$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_travis.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY) - ./scripts/clean_fcctr.sh + sudo mkdir -m777 -p $(CTRDLOGDIR) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.err" \ + go test $(EXTRAGOARGS_NORACE) -run TestParallelServe + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.err" \ + go test $(EXTRAGOARGS) -run TestServeThree + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.err" \ + go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_both_log_man_travis.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_both_log_man_travis.err" \ + go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) $(WITHUPF) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_travis.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_travis.err" \ + go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY) test-skip: sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_skip.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_skip.err & @@ -92,25 +166,55 @@ test-skip: bench: ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy - ./scripts/clean_fcctr.sh + sudo mkdir -m777 -p $(CTRDLOGDIR) + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP - ./scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy - ./scripts/clean_fcctr.sh + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP + + sudo env \ + "PATH=$(PATH)" \ + "FIRECRACKER_BIN=$(FIRECRACKER_BIN)" \ + "FIRECRACKER_CONFIG_PATH=$(FIRECRACKER_CONFIG_PATH)" \ + "LOG_OUT=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out" \ + "LOG_ERR=$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err" \ + go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy test-man-bench: $(MAKE) test-man diff --git a/bench_test.go b/bench_test.go index 026ea14b1..0948d2d60 100644 --- a/bench_test.go +++ b/bench_test.go @@ -48,6 +48,11 @@ var ( ) func TestBenchParallelServe(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + var ( servedTh uint64 pinnedFuncNum int @@ -124,6 +129,11 @@ func TestBenchParallelServe(t *testing.T) { } func TestBenchWarmServe(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + var ( servedTh uint64 pinnedFuncNum int @@ -189,6 +199,11 @@ func TestBenchWarmServe(t *testing.T) { } func TestBenchServe(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + var ( servedTh uint64 pinnedFuncNum int diff --git a/manual_cleanup_test.go b/manual_cleanup_test.go index cb51efc0f..0815be41d 100644 --- a/manual_cleanup_test.go +++ b/manual_cleanup_test.go @@ -33,6 +33,11 @@ import ( ) func TestParallelServe(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + var ( servedTh uint64 = 1 pinnedFuncNum int @@ -67,6 +72,11 @@ func TestParallelServe(t *testing.T) { } func TestServeThree(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "200" var ( servedTh uint64 = 1 diff --git a/perf_bench_test.go b/perf_bench_test.go index 9ae8a5873..af9da2913 100644 --- a/perf_bench_test.go +++ b/perf_bench_test.go @@ -88,6 +88,11 @@ func TestProfileIncrementConfiguration(t *testing.T) { t.Skip("Skipping TestProfileIncrementConfiguration") + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + var ( idx, rps int pinnedFuncNum int @@ -136,6 +141,11 @@ func TestProfileSingleConfiguration(t *testing.T) { t.Skip("Skipping TestProfileSingleConfiguration") + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + var ( servedTh uint64 pinnedFuncNum int @@ -211,6 +221,11 @@ func TestColocateVMsOnSameCPU(t *testing.T) { } func TestBindSocket(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + var ( procStr, sep string servedTh uint64 diff --git a/vhive_test.go b/vhive_test.go index 9a7947f84..1340ac91c 100644 --- a/vhive_test.go +++ b/vhive_test.go @@ -25,15 +25,19 @@ package main import ( "context" "flag" + "fmt" + "net" "os" + "os/exec" "strconv" "sync" "testing" + "time" ctrdlog "github.com/containerd/containerd/log" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" - ctriface "github.com/vhive-serverless/vhive/ctriface" + "github.com/vhive-serverless/vhive/ctriface" ) const ( @@ -77,29 +81,16 @@ func TestMain(m *testing.M) { log.Infof("Drop cache: %t", !*isWithCache) log.Infof("Bench dir: %s", *benchDir) - orch = ctriface.NewOrchestrator( - "devmapper", - "", - ctriface.WithTestModeOn(true), - ctriface.WithSnapshots(*isSnapshotsEnabledTest), - ctriface.WithUPF(*isUPFEnabledTest), - ctriface.WithMetricsMode(*isMetricsModeTest), - ctriface.WithLazyMode(*isLazyModeTest), - ) - ret := m.Run() - - err := orch.StopActiveVMs() - if err != nil { - log.Printf("Failed to stop VMs, err: %v\n", err) - } - - orch.Cleanup() - os.Exit(ret) } func TestSendToFunctionSerial(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "1" var ( servedTh uint64 @@ -122,6 +113,11 @@ func TestSendToFunctionSerial(t *testing.T) { } func TestSendToFunctionParallel(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "2" var ( servedTh uint64 @@ -148,6 +144,11 @@ func TestSendToFunctionParallel(t *testing.T) { } func TestStartSendStopTwice(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "3" var ( servedTh uint64 = 1 @@ -173,6 +174,11 @@ func TestStartSendStopTwice(t *testing.T) { } func TestStatsNotNumericFunction(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "not-cld" var ( servedTh uint64 = 1 @@ -194,6 +200,11 @@ func TestStatsNotNumericFunction(t *testing.T) { } func TestStatsNotColdFunction(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "4" var ( servedTh uint64 = 1 @@ -215,6 +226,11 @@ func TestStatsNotColdFunction(t *testing.T) { } func TestSaveMemorySerial(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "5" var ( servedTh uint64 = 40 @@ -236,6 +252,11 @@ func TestSaveMemorySerial(t *testing.T) { } func TestSaveMemoryParallel(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "6" var ( servedTh uint64 = 40 @@ -266,6 +287,11 @@ func TestSaveMemoryParallel(t *testing.T) { } func TestDirectStartStopVM(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() + fID := "7" var ( servedTh uint64 @@ -285,6 +311,10 @@ func TestDirectStartStopVM(t *testing.T) { } func TestAllFunctions(t *testing.T) { + if err := initFirecrackerContainerd(); err != nil { + t.Fatalf("Failed to initialize firecracker containerd: %v", err) + } + defer cleanup() if testing.Short() { t.Skip("skipping TestAllFunctions in non-nightly runs.") @@ -340,3 +370,78 @@ func TestAllFunctions(t *testing.T) { } vmGroup.Wait() } + +func initFirecrackerContainerd() error { + firecrackerBin := os.Getenv("FIRECRACKER_BIN") + firecrackerConfigPath := os.Getenv("FIRECRACKER_CONFIG_PATH") + logOut := os.Getenv("LOG_OUT") + logErr := os.Getenv("LOG_ERR") + + logOutFile, err := os.OpenFile(logOut, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return fmt.Errorf("failed to open log file %s: %w", logOut, err) + } + defer logOutFile.Close() + + logErrFile, err := os.OpenFile(logErr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return fmt.Errorf("failed to open error log file %s: %w", logErr, err) + } + defer logErrFile.Close() + + cmd := exec.Command(firecrackerBin, "--config", firecrackerConfigPath) + cmd.Stdout = logOutFile + cmd.Stderr = logErrFile + + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start firecracker containerd: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := waitForContainerd(ctx); err != nil { + return fmt.Errorf("failed to connect to containerd: %w", err) + } + + orch = ctriface.NewOrchestrator( + "devmapper", + "", + ctriface.WithTestModeOn(true), + ctriface.WithSnapshots(*isSnapshotsEnabledTest), + ctriface.WithUPF(*isUPFEnabledTest), + ctriface.WithMetricsMode(*isMetricsModeTest), + ctriface.WithLazyMode(*isLazyModeTest), + ) + + return nil +} + +func waitForContainerd(ctx context.Context) error { + containerdAddress := "/run/firecracker-containerd/containerd.sock" + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if conn, err := net.DialTimeout("unix", containerdAddress, 200*time.Millisecond); err == nil { + conn.Close() + return nil + } + } + } +} + +func cleanup() { + if err := orch.StopActiveVMs(); err != nil { + log.Printf("Failed to stop VMs, err: %v\n", err) + } + orch.Cleanup() + err := exec.Command("bash", "scripts/clean_fcctr.sh").Run() + if err != nil { + log.Printf("Failed to clean up firecracker containerd, err: %v\n", err) + } +}