proxy: implement (re)store of proxy's state

Introduce the high availability feature of cc-proxy by implementing store/restore of proxy's state to/from disk. This feature depends on the ability of shim to reconnect to cc-proxy if connection is lost. Fixes clearcontainers#4. Signed-off-by: Dmitry Voytik <[email protected]>
dvoytik · Nov 7, 2017 · b8fbc2e · b8fbc2e
1 parent 32cad21
commit b8fbc2e
Show file tree

Hide file tree

Showing 5 changed files with 843 additions and 10 deletions.
diff --git a/Makefile b/Makefile
@@ -16,6 +16,7 @@ LOCALSTATEDIR := /var
 
 SOURCES := $(shell find . 2>&1 | grep -E '.*\.(c|h|go)$$')
 PROXY_SOCKET := $(LOCALSTATEDIR)/run/clear-containers/proxy.sock
+STORE_STATE_DIR := $(LOCALSTATEDIR)/run/clear-containers/proxy/
 
 DESCRIBE := $(shell git describe 2> /dev/null || true)
 DESCRIBE_DIRTY := $(if $(shell git status --porcelain --untracked-files=no 2> /dev/null),${DESCRIBE}-dirty,${DESCRIBE})
@@ -53,7 +54,7 @@ all: cc-proxy $(UNIT_FILES)
 
 cc-proxy: $(SOURCES) Makefile
 	$(QUIET_GOBUILD)go build -i -o $@ -ldflags \
-		"-X main.DefaultSocketPath=$(PROXY_SOCKET) -X main.Version=$(VERSION)"
+		"-X main.DefaultSocketPath=$(PROXY_SOCKET) -X main.Version=$(VERSION) -X main.storeStateDir=$(STORE_STATE_DIR)"
 
 #
 # Tests

diff --git a/proxy.go b/proxy.go
@@ -255,6 +255,15 @@ func registerVM(data []byte, userData interface{}, response *handlerResponse) {
 
 	client.vm = vm
 
+	if err := storeVMState(vm); err != nil {
+		logContID(vm.containerID).Errorf(
+			"couldn't store a VM state: %v", err)
+	}
+
+	if err := storeProxyState(proxy); err != nil {
+		proxyLog.Errorf("couldn't store proxy's state: %v", err)
+	}
+
 	if proxyKSM != nil {
 		proxyKSM.kick()
 	}
@@ -300,6 +309,15 @@ func attachVM(data []byte, userData interface{}, response *handlerResponse) {
 	client.log.Infof("AttachVM(containerId=%s)", payload.ContainerID)
 
 	client.vm = vm
+
+	if err := storeVMState(vm); err != nil {
+		logContID(vm.containerID).Errorf(
+			"couldn't store a VM state: %v", err)
+	}
+
+	if err := storeProxyState(proxy); err != nil {
+		proxyLog.Errorf("couldn't store proxy's state: %v", err)
+	}
 }
 
 // "UnregisterVM"
@@ -330,9 +348,10 @@ func unregisterVM(data []byte, userData interface{}, response *handlerResponse)
 
 	client.log.Info("UnregisterVM()")
 
-	proxy.Lock()
-	delete(proxy.vms, vm.containerID)
-	proxy.Unlock()
+	if err := delVMAndState(proxy, vm); err != nil {
+		logContID(payload.ContainerID).Warnf("Error deleting state: %v",
+			err)
+	}
 
 	client.vm = nil
 }
@@ -627,12 +646,19 @@ func (proxy *proxy) init() error {
 	// Force a coredump + full stacktrace on internal error
 	debug.SetTraceback("crash")
 
-	// flags
-	proxy.enableVMConsole = logrus.GetLevel() == logrus.DebugLevel
+	stateIsRestored, err := restoreAllState(proxy)
+	if err != nil {
+		proxyLog.Errorf("Restoring failed: %v", err)
+	}
+
+	if !stateIsRestored {
+		// flags
+		proxy.enableVMConsole = logrus.GetLevel() == logrus.DebugLevel
 
-	// Open the proxy socket
-	if proxy.socketPath, err = getSocketPath(); err != nil {
-		return fmt.Errorf("couldn't get a rigth socket path: %v", err)
+		// Open the proxy socket
+		if proxy.socketPath, err = getSocketPath(); err != nil {
+			return fmt.Errorf("couldn't get a right socket path: %v", err)
+		}
 	}
 	fds := listenFds()