diff --git a/cmd/internal/api/versions.go b/cmd/internal/api/versions.go index 7577c91cab..8c0810ec7c 100644 --- a/cmd/internal/api/versions.go +++ b/cmd/internal/api/versions.go @@ -531,6 +531,7 @@ func GetRequestOptions(r *http.Request) (v2.RequestOptions, error) { supportedTypes := map[string]bool{ v2.TypeName: true, v2.TypeDocker: true, + v2.TypePodman: true, } // fill in the defaults. opt := v2.RequestOptions{ diff --git a/cmd/internal/container/install/install.go b/cmd/internal/container/install/install.go index d35db81047..5d3153c727 100644 --- a/cmd/internal/container/install/install.go +++ b/cmd/internal/container/install/install.go @@ -21,5 +21,6 @@ import ( _ "github.com/google/cadvisor/container/containerd/install" _ "github.com/google/cadvisor/container/crio/install" _ "github.com/google/cadvisor/container/docker/install" + _ "github.com/google/cadvisor/container/podman/install" _ "github.com/google/cadvisor/container/systemd/install" ) diff --git a/cmd/internal/pages/assets/html/containers.html b/cmd/internal/pages/assets/html/containers.html index 8c4643b553..c49f1e6fdf 100644 --- a/cmd/internal/pages/assets/html/containers.html +++ b/cmd/internal/pages/assets/html/containers.html @@ -50,6 +50,9 @@

{{.DisplayName}}

Docker Containers

+
+

Podman Containers

+
{{end}} {{if .Subcontainers}}
@@ -250,8 +253,7 @@

Top Memory Usage: class="subcontainer-display-input" value=10>

- - +
diff --git a/cmd/internal/pages/docker.go b/cmd/internal/pages/docker.go index 36fb960516..c512b83af6 100644 --- a/cmd/internal/pages/docker.go +++ b/cmd/internal/pages/docker.go @@ -23,6 +23,7 @@ import ( "time" "github.com/google/cadvisor/container/docker" + dockerutil "github.com/google/cadvisor/container/docker/utils" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/manager" @@ -39,12 +40,12 @@ func toStatusKV(status info.DockerStatus) ([]keyVal, []keyVal) { ds = append(ds, keyVal{Key: k, Value: v}) } return []keyVal{ - {Key: "Docker Version", Value: status.Version}, - {Key: "Docker API Version", Value: status.APIVersion}, + {Key: "Version", Value: status.Version}, + {Key: "API Version", Value: status.APIVersion}, {Key: "Kernel Version", Value: status.KernelVersion}, {Key: "OS Version", Value: status.OS}, {Key: "Host Name", Value: status.Hostname}, - {Key: "Docker Root Directory", Value: status.RootDir}, + {Key: "Root Directory", Value: status.RootDir}, {Key: "Execution Driver", Value: status.ExecDriver}, {Key: "Number of Images", Value: strconv.Itoa(status.NumImages)}, {Key: "Number of Containers", Value: strconv.Itoa(status.NumContainers)}, @@ -73,7 +74,7 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) { for _, cont := range conts { subcontainers = append(subcontainers, link{ Text: getContainerDisplayName(cont.ContainerReference), - Link: path.Join(rootDir, DockerPage, docker.ContainerNameToDockerId(cont.ContainerReference.Name)), + Link: path.Join(rootDir, DockerPage, dockerutil.ContainerNameToId(cont.ContainerReference.Name)), }) } @@ -126,7 +127,7 @@ func serveDockerPage(m manager.Manager, w http.ResponseWriter, u *url.URL) { }) parentContainers = append(parentContainers, link{ Text: displayName, - Link: path.Join(rootDir, DockerPage, docker.ContainerNameToDockerId(cont.Name)), + Link: path.Join(rootDir, DockerPage, dockerutil.ContainerNameToId(cont.Name)), }) // Get the MachineInfo diff --git a/cmd/internal/pages/pages.go b/cmd/internal/pages/pages.go index 75c0bf7d76..516f6dea1a 100644 --- a/cmd/internal/pages/pages.go +++ b/cmd/internal/pages/pages.go @@ -99,15 +99,29 @@ func dockerHandler(containerManager manager.Manager) auth.AuthenticatedHandlerFu } } +func podmanHandlerNoAuth(containerManager manager.Manager) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + servePodmanPage(containerManager, w, r.URL) + } +} + +func podmanHandler(containerManager manager.Manager) auth.AuthenticatedHandlerFunc { + return func(w http.ResponseWriter, r *auth.AuthenticatedRequest) { + servePodmanPage(containerManager, w, r.URL) + } +} + // Register http handlers func RegisterHandlersDigest(mux httpmux.Mux, containerManager manager.Manager, authenticator *auth.DigestAuth, urlBasePrefix string) error { // Register the handler for the containers page. if authenticator != nil { mux.HandleFunc(ContainersPage, authenticator.Wrap(containerHandler(containerManager))) mux.HandleFunc(DockerPage, authenticator.Wrap(dockerHandler(containerManager))) + mux.HandleFunc(PodmanPage, authenticator.Wrap(podmanHandler(containerManager))) } else { mux.HandleFunc(ContainersPage, containerHandlerNoAuth(containerManager)) mux.HandleFunc(DockerPage, dockerHandlerNoAuth(containerManager)) + mux.HandleFunc(PodmanPage, podmanHandlerNoAuth(containerManager)) } if ContainersPage[len(ContainersPage)-1] == '/' { @@ -118,6 +132,10 @@ func RegisterHandlersDigest(mux httpmux.Mux, containerManager manager.Manager, a redirectHandler := http.RedirectHandler(urlBasePrefix+DockerPage, http.StatusMovedPermanently) mux.Handle(DockerPage[0:len(DockerPage)-1], redirectHandler) } + if PodmanPage[len(PodmanPage)-1] == '/' { + redirectHandler := http.RedirectHandler(urlBasePrefix+PodmanPage, http.StatusMovedPermanently) + mux.Handle(PodmanPage[0:len(PodmanPage)-1], redirectHandler) + } return nil } @@ -127,9 +145,11 @@ func RegisterHandlersBasic(mux httpmux.Mux, containerManager manager.Manager, au if authenticator != nil { mux.HandleFunc(ContainersPage, authenticator.Wrap(containerHandler(containerManager))) mux.HandleFunc(DockerPage, authenticator.Wrap(dockerHandler(containerManager))) + mux.HandleFunc(PodmanPage, authenticator.Wrap(podmanHandler(containerManager))) } else { mux.HandleFunc(ContainersPage, containerHandlerNoAuth(containerManager)) mux.HandleFunc(DockerPage, dockerHandlerNoAuth(containerManager)) + mux.HandleFunc(PodmanPage, podmanHandlerNoAuth(containerManager)) } if ContainersPage[len(ContainersPage)-1] == '/' { diff --git a/cmd/internal/pages/podman.go b/cmd/internal/pages/podman.go new file mode 100644 index 0000000000..4f29d1141a --- /dev/null +++ b/cmd/internal/pages/podman.go @@ -0,0 +1,138 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pages + +import ( + "fmt" + "net/http" + "net/url" + "path" + "time" + + dockerutil "github.com/google/cadvisor/container/docker/utils" + "github.com/google/cadvisor/container/podman" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/manager" + + "k8s.io/klog/v2" +) + +const PodmanPage = "/podman/" + +func servePodmanPage(m manager.Manager, w http.ResponseWriter, u *url.URL) { + start := time.Now() + + containerName := u.Path[len(PodmanPage)-1:] + rootDir := getRootDir(containerName) + + var data *pageData + + if containerName == "/" { + // Scenario for all containers. + status, err := podman.Status() + if err != nil { + http.Error(w, fmt.Sprintf("failed to get podman info: %v", err), http.StatusInternalServerError) + return + } + images, err := podman.Images() + if err != nil { + http.Error(w, fmt.Sprintf("failed to get podman images: %v", err), http.StatusInternalServerError) + return + } + + reqParams := info.ContainerInfoRequest{ + NumStats: 0, + } + conts, err := m.AllPodmanContainers(&reqParams) + if err != nil { + http.Error(w, fmt.Sprintf("failed to get container %q with error: %v", containerName, err), http.StatusNotFound) + return + } + subcontainers := make([]link, 0, len(conts)) + for _, cont := range conts { + subcontainers = append(subcontainers, link{ + Text: getContainerDisplayName(cont.ContainerReference), + Link: path.Join(rootDir, PodmanPage, dockerutil.ContainerNameToId(cont.ContainerReference.Name)), + }) + } + + podmanStatus, driverStatus := toStatusKV(status) + + podmanContainerText := "Podman Containers" + data = &pageData{ + DisplayName: podmanContainerText, + ParentContainers: []link{ + { + Text: podmanContainerText, + Link: path.Join(rootDir, PodmanPage), + }}, + Subcontainers: subcontainers, + Root: rootDir, + DockerStatus: podmanStatus, + DockerDriverStatus: driverStatus, + DockerImages: images, + } + } else { + // Scenario for specific container. + machineInfo, err := m.GetMachineInfo() + if err != nil { + http.Error(w, fmt.Sprintf("failed to get machine info: %v", err), http.StatusInternalServerError) + return + } + + reqParams := info.ContainerInfoRequest{ + NumStats: 60, + } + cont, err := m.PodmanContainer(containerName[1:], &reqParams) + if err != nil { + http.Error(w, fmt.Sprintf("failed to get container %v with error: %v", containerName, err), http.StatusNotFound) + return + } + displayName := getContainerDisplayName(cont.ContainerReference) + + var parentContainers []link + parentContainers = append(parentContainers, link{ + Text: "Podman Containers", + Link: path.Join(rootDir, PodmanPage), + }) + parentContainers = append(parentContainers, link{ + Text: displayName, + Link: path.Join(rootDir, PodmanPage, dockerutil.ContainerNameToId(cont.Name)), + }) + + data = &pageData{ + DisplayName: displayName, + ContainerName: escapeContainerName(cont.Name), + ParentContainers: parentContainers, + Spec: cont.Spec, + Stats: cont.Stats, + MachineInfo: machineInfo, + ResourcesAvailable: cont.Spec.HasCpu || cont.Spec.HasMemory || cont.Spec.HasNetwork, + CpuAvailable: cont.Spec.HasCpu, + MemoryAvailable: cont.Spec.HasMemory, + NetworkAvailable: cont.Spec.HasNetwork, + FsAvailable: cont.Spec.HasFilesystem, + CustomMetricsAvailable: cont.Spec.HasCustomMetrics, + Root: rootDir, + } + } + + err := pageTemplate.Execute(w, data) + if err != nil { + klog.Errorf("Failed to apply template: %s", err) + } + + klog.V(5).Infof("Request took %s", time.Since(start)) +} diff --git a/cmd/internal/pages/templates.go b/cmd/internal/pages/templates.go index 9674db7187..568aa09ec2 100644 --- a/cmd/internal/pages/templates.go +++ b/cmd/internal/pages/templates.go @@ -1,4 +1,4 @@ -// Copyright 2022 Google Inc. All Rights Reserved. +// Copyright 2023 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ // Code generated by go-bindata. DO NOT EDIT. // sources: -// cmd/internal/pages/assets/html/containers.html (9.554kB) +// cmd/internal/pages/assets/html/containers.html (9.64kB) package pages @@ -83,7 +83,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _cmdInternalPagesAssetsHtmlContainersHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x5a\x4d\x73\xdb\x38\xd2\x3e\x4b\xbf\xa2\x87\xf5\x1e\x66\xaa\x42\xca\x4e\xfc\x1e\x36\x2b\xab\x4a\xa3\x24\x3b\xda\x71\xec\x94\x65\xcf\xd4\x1c\x41\xb2\x45\x22\x86\x08\x0c\x00\x4a\xd6\xba\xfc\xdf\xb7\x00\x90\x12\x3f\xa5\xf8\xa3\x92\xd5\xc5\x12\x81\xee\x7e\xfa\xe9\x6e\xa0\x41\x78\xfc\x93\xef\x0f\x01\x66\x5c\x6c\x25\x4d\x52\x0d\x6f\x4f\x4e\xcf\xe0\x5f\x9c\x27\x0c\x61\x9e\x45\x01\x4c\x19\x83\x6b\x33\xa4\xe0\x1a\x15\xca\x35\xc6\xc1\x70\x08\x70\x41\x23\xcc\x14\xc6\x90\x67\x31\x4a\xd0\x29\xc2\x54\x90\x28\xc5\x72\xe4\x0d\xfc\x81\x52\x51\x9e\xc1\xdb\xe0\x04\x7e\x36\x13\xbc\x62\xc8\xfb\xe5\x9f\x43\x80\x2d\xcf\x61\x45\xb6\x90\x71\x0d\xb9\x42\xd0\x29\x55\xb0\xa4\x0c\x01\xef\x23\x14\x1a\x68\x06\x11\x5f\x09\x46\x49\x16\x21\x6c\xa8\x4e\xad\x99\x42\x49\x30\x04\xf8\xab\x50\xc1\x43\x4d\x68\x06\x04\x22\x2e\xb6\xc0\x97\xd5\x79\x40\xb4\xc1\x6b\x3e\xa9\xd6\xe2\xfd\x68\xb4\xd9\x6c\x02\x62\xb1\x06\x5c\x26\x23\xe6\xe6\xa9\xd1\xc5\x7c\xf6\xf1\x72\xf1\xd1\x7f\x1b\x9c\x18\x89\xdb\x8c\xa1\x52\x20\xf1\xef\x9c\x4a\x8c\x21\xdc\x02\x11\x82\xd1\x88\x84\x0c\x81\x91\x0d\x70\x09\x24\x91\x88\x31\x68\x6e\xd0\x6e\x24\xd5\x34\x4b\xde\x80\xe2\x4b\xbd\x21\x12\x87\x00\x31\x55\x5a\xd2\x30\xd7\x35\xaa\x4a\x6c\x54\xd5\x26\xf0\x0c\x48\x06\xde\x74\x01\xf3\x85\x07\xbf\x4e\x17\xf3\xc5\x9b\x21\xc0\x9f\xf3\x9b\xdf\xae\x6e\x6f\xe0\xcf\xe9\xf5\xf5\xf4\xf2\x66\xfe\x71\x01\x57\xd7\x30\xbb\xba\xfc\x30\xbf\x99\x5f\x5d\x2e\xe0\xea\x13\x4c\x2f\xff\x82\xdf\xe7\x97\x1f\xde\x00\x52\x9d\xa2\x04\xbc\x17\xd2\xe0\xe7\x12\xa8\x21\xd1\xc4\x0d\x60\x81\x58\x03\xb0\xe4\x0e\x90\x12\x18\xd1\x25\x8d\x80\x91\x2c\xc9\x49\x82\x90\xf0\x35\xca\x8c\x66\x09\x08\x94\x2b\xaa\x4c\x28\x15\x90\x2c\x1e\x02\x30\xba\xa2\x9a\x68\xfb\xa4\xe5\x54\x30\xf4\xfd\xc9\x70\x38\x4e\xf5\x8a\x4d\x86\x00\xe3\x14\x49\x3c\xb1\x21\x18\x6b\xaa\x19\x4e\xa2\x69\xbc\xa6\x8a\x4b\xf0\xe1\xe1\x21\xf8\x40\x95\x60\x64\x7b\x49\x56\xf8\xf8\x38\x1e\xb9\x29\x6e\xba\x8a\x24\x15\x1a\x94\x8c\xce\xbd\x87\x87\xe0\x9a\x73\xfd\xf8\xa8\x8c\xe5\x68\x24\xb8\x10\x28\x83\x15\xcd\x82\xaf\xca\x9b\x8c\x47\x6e\x72\x21\xf9\x93\xef\xc3\x05\xd1\xa8\xb4\xcd\x21\xca\x30\x36\xd8\x61\x45\x33\xba\xa4\x18\xc3\x6c\xb1\x00\x83\xd3\xce\x66\x34\xbb\x03\x89\xec\xdc\x53\x7a\xcb\x50\xa5\x88\xda\x83\x54\xe2\xb2\x6d\x37\xe4\x5c\x2b\x2d\x89\xf0\xcf\x82\x93\xe0\xc4\x0f\x51\x93\xe0\xad\xc5\x11\x29\xe5\x4d\x86\x7b\x00\x57\xc2\x50\x44\x98\x61\x67\x85\x2f\x35\x67\x95\xf8\xef\x82\xd3\xe0\xb4\x65\xed\x29\x1a\x23\x9e\x99\x6a\x41\xa9\x5a\x80\x0f\x32\xf6\x6f\xb2\x26\x0b\x17\x90\x9d\x27\x87\x02\xf4\xf5\xef\x1c\xe5\xd6\x7f\x17\xfc\x7f\x01\xb8\x23\x4c\x87\xe4\x0f\x10\xdd\xd6\xb4\xd7\xa5\xb7\x02\xcf\x3d\x8d\xf7\x7a\xf4\x95\xac\x89\x7b\xea\x75\x9b\x60\x9c\xc4\x28\x0f\x00\x7b\x8a\xb2\x0a\xaf\x4d\x85\xe3\x51\x59\x03\xe3\x90\xc7\xdb\xc2\x46\x4c\xd7\x10\x31\xa2\xd4\xb9\xb7\x93\x75\xa9\xe2\xab\x94\x6f\x22\xa2\xd0\x83\x9d\x7b\xa4\x19\x4e\x6f\x2f\xcc\x7c\xb5\xf2\x4f\xdf\x7a\x40\xe3\x73\x8f\xf1\x84\x7b\x3b\xb1\x11\xd9\x7d\xad\xd9\x2b\x45\x26\xc3\x41\x75\x40\x90\x04\x7d\x03\x16\xa5\x19\x32\xd5\x7b\x3a\x69\x17\x69\x7a\x6a\xe4\x46\x31\x5d\x9b\xbf\x9c\x95\xe2\xa1\x44\x12\x47\x32\x5f\x85\x4e\xfa\xe1\x41\x92\x2c\x41\xf8\x3f\x41\x24\x66\x7a\xb6\x73\xf3\xfd\x39\x04\x5f\xea\xcf\xd4\xe3\xa3\x35\xc8\xe8\xa4\xe2\x6c\x53\x32\xb8\xa0\xd9\xdd\xe3\xa3\x37\xe9\x18\xba\xc1\x7b\x6d\xd0\x91\xc9\x78\xc4\x68\x01\x00\xb3\xd8\x28\x1e\x8f\x38\xdb\x93\x62\x81\xbb\x1f\x0f\x0f\x74\x09\xc1\x5c\x39\x52\x8f\x70\x05\xc5\x67\x9c\x9e\xed\x41\x06\xc1\x28\xe6\xd1\x9d\x61\xec\x83\xfd\x0b\x7b\x9f\x1c\x98\xf4\xac\xc7\xb4\x03\x57\x05\xb2\xc8\xc3\xa8\xca\xc8\xcb\x62\xf7\x6e\x52\xd3\x37\x1e\xa5\xef\xaa\x81\xab\x08\x33\xaa\xb4\x9f\x48\x9e\x8b\x46\xe4\x54\x45\x81\x0d\x5b\x13\xe1\xa0\x96\x9c\xb5\xf9\x65\xb0\xda\x46\x7c\xaa\x71\x65\x83\x58\x9b\xbf\x8f\x60\x23\x78\x15\xd6\xfa\x29\x74\x0c\xba\x18\x2c\x34\xd1\xf9\x6b\x10\xf8\x41\xd2\x35\x4a\x70\xfa\x9a\x04\xe6\xec\x28\x7f\x2e\x35\x94\x15\xb7\xfc\x35\xf0\xb9\x94\x77\x6a\xa0\x83\xa2\xb1\x12\x24\x2b\xad\x18\x35\x3e\x23\x21\x32\xcb\x5d\x55\x77\xf0\x3b\x6e\x0d\x75\x66\xfa\x04\x9a\x83\x7f\x10\x96\xdb\xca\x6d\xd6\x45\x9d\x35\xe7\xec\x1e\xdb\xe0\x79\xd0\x16\x9a\x4b\x92\xe0\x38\x94\x93\x02\x90\x51\xd5\x47\xd6\x60\xcf\x95\x35\xdf\xe2\xaa\x1f\xd5\x53\xf9\xaa\xe8\x6f\xf3\x55\x1d\xac\xf3\x35\xd8\xd1\x35\x18\x8f\x72\x66\xbd\x29\x99\x2c\x1e\xf4\x65\x6b\x57\x8d\x3b\xaf\xe6\x2b\x92\xe0\xf1\x0c\x85\xdd\xa7\x3f\x55\xa1\xf2\x31\x39\xeb\x54\xbb\x64\xad\x8c\x54\x71\x39\x6d\x66\xbf\x70\x79\xe2\x53\x2b\x63\xf6\xad\xda\x2c\x13\xc2\x50\xee\x7f\x1f\xf3\xed\x1a\x15\xcf\x65\x84\x6a\xba\x26\x94\x99\x56\xf9\x15\x6a\x70\xae\x38\xb3\xed\x66\xa3\xfe\x9c\xc9\x99\xc8\xab\xc6\x7a\x13\xad\xc2\x44\x6f\xfe\x00\x89\x34\x5d\x9b\xc6\xbc\xb0\xe8\xdb\x7e\x14\x04\xc9\x90\xb9\xef\xde\x64\xf6\xe5\xd6\x85\x7f\xaf\xb1\x58\xbc\x05\x46\x06\x4e\x70\x61\x1a\xe4\x9d\xe3\x87\x4d\x1e\xaa\xa3\x94\x48\x13\xc7\x32\x47\x85\xa4\x99\x76\x0f\xdb\xc6\xa0\xa6\x26\xcf\xe8\x4e\x8d\xaa\xaa\x69\x23\xaf\x06\xb1\xc3\x97\xcf\xe4\xfe\x95\xdc\xf9\x4c\xee\xc1\xaa\x6a\x78\x34\xe3\x75\x87\xf6\x16\xfb\x7d\x8a\xf8\x8b\x5c\x52\x77\x2f\x77\x67\xca\x18\xdf\x98\xa3\x04\x6f\x07\xc9\x58\x68\x18\x84\xe0\x33\x89\x52\x9a\xe1\x3c\x5b\xf2\xe0\x32\x5f\x59\xb9\x72\x8d\x69\xa3\x2f\x97\x9a\xdd\x6f\xe7\xc4\x67\x5c\x71\xb9\xfd\xbe\x09\xef\x6c\x1e\xc8\x79\x37\x21\x70\x6f\x08\xac\x9a\x97\xd3\x5b\x51\xd6\xac\x00\xfa\x1f\x3c\x60\xb8\x3f\x69\x0a\xf9\xdb\x8c\xea\x03\xf2\xcf\xc9\xaa\x42\xcf\x2b\x15\x4a\x57\x91\xb4\x9d\x3e\x5a\x23\xbd\xee\x16\x92\x2f\x70\x74\xb1\x21\xe2\xb5\x16\xb9\x0d\x11\x9d\xcb\x42\xdb\xe3\x8a\xd5\x67\x78\x5d\x91\x3e\xe2\x79\xb3\xf4\x0a\xef\x6a\x5d\xe8\xb3\x37\xb3\x5b\x65\x5a\xa3\xfe\x4e\xdc\x56\x5e\x51\x7f\x42\xd2\x15\x91\xdb\x03\x6d\x80\x99\x65\x2c\xd0\x2c\x69\x37\x02\xf5\x69\x45\x31\x5f\xad\x51\xae\x29\x6e\x0e\xb7\x07\xd5\x0e\x21\x37\x88\xfd\x84\xe4\x09\x7a\x75\x95\xe6\x34\xbb\x6b\x19\x7e\x88\x37\x5f\x24\x8f\x50\xa9\x63\xdd\x4e\xd5\x1d\x51\x8a\xf8\x9a\x8b\x6f\x72\xa8\xa7\xcf\xf8\x8e\x6e\xda\x96\xe3\x5b\x1c\xec\xf0\xa6\x61\xe0\x6c\x72\xc3\x35\x61\x50\xe6\xe1\x99\xcd\xcc\x0a\x3f\x91\xc8\x7d\x6d\xa6\xf8\x2e\xf0\x51\x4a\xa4\xde\x93\x02\xe5\xdb\x22\xa3\x6a\xf6\xe5\x16\x2e\x38\x89\x61\xba\x46\x79\x40\x1f\xe3\x24\xae\x2b\xda\xbd\x44\xaa\x22\xb3\x98\x40\xd8\x23\xb4\xec\x55\x26\x50\xfa\x66\xff\xef\xc4\xd7\xad\xf2\x57\x89\xe4\x2e\xe6\x9b\xac\x4f\xa7\x53\x15\x96\xd3\x7a\x95\xb6\x53\xe3\xe8\xee\xfc\x1d\xd3\xa4\xdc\xa8\xbf\x53\xa6\xac\xac\xb9\xe3\x61\x08\xe5\xa8\xf1\xa4\x02\x40\xf2\x0d\x74\x1f\x78\x0e\x86\xb0\x31\xad\xbd\x1c\xff\xc3\x9e\x2d\x6b\xae\x4a\x9e\x48\xb4\xef\x3c\xa1\xf5\xe9\x9a\xe8\x87\x44\x42\xf5\x87\x1f\x9b\x83\xaa\xf4\xca\x75\xc4\x0d\xa4\x5c\xfb\x8e\x8a\x4e\xcd\x50\xdf\xab\x94\xf4\x79\xc6\xb6\xde\xe4\x37\xae\xa1\x0c\x98\x3b\x24\x77\x48\xb6\xd9\x7c\x0a\x5c\x9a\x2d\x79\x03\x6c\xc4\x59\xfc\x1c\xb4\x33\xce\xe2\x6f\x85\x3b\x18\x74\xe2\xee\x7e\xd8\x8e\xdc\x3b\xaf\x9a\x5d\x1a\xef\x9b\xab\xcf\x13\x8b\xf2\x12\xf5\x86\xcb\xbb\x27\x56\xe5\xe0\xe5\xe5\x58\x18\x2e\x36\xfb\xa7\x14\xe2\xa0\x39\x1a\x4b\x2e\x4c\xf2\xb7\x0b\x24\xcc\xb5\xe6\xbb\x78\x85\x3a\x83\x50\x67\x7e\x8c\x4b\x92\x33\x0d\xa5\x9c\xaf\x79\x92\x30\xf4\x8a\x57\xda\x4e\xc8\xf1\x9c\x39\x94\xbe\x42\x86\x91\x3d\x02\xec\x8c\x41\x4c\x34\x29\x44\x2b\x18\x80\x48\x4a\xfc\x94\x28\xc1\x45\x2e\xce\x3d\x2d\x73\x2c\x1e\xe2\xbd\x20\x59\x8c\xf1\xb9\xb7\x24\x4c\x61\x47\x8a\xb9\xf4\xea\x36\x5c\xc6\xba\x3b\xbf\x6a\x89\x19\x11\x89\x95\xb9\x83\x32\x13\x9c\x67\x2d\x96\x72\xd6\x6d\xd2\x6b\x12\xec\xaf\x30\xcb\x3d\x90\xdc\x78\xec\xbe\x5b\xc7\x6c\x77\xc9\x30\x0e\xb7\x07\x19\x6b\xe7\x7c\xf1\x7a\xe8\x40\xda\x3e\x65\x41\x4e\x25\xcf\x93\x54\xe4\xba\xbd\x0a\xee\x96\xe5\x12\x5e\xb8\xd5\xa8\xda\xdb\xf7\x33\xcc\x7e\x94\x92\xdb\xd7\xc7\xad\x2d\xa0\xb4\x85\x76\x46\xbf\xb1\x86\xf3\x8d\x0a\xfd\xa4\x7e\xd8\x96\xf9\x89\x32\x54\x5b\xa5\x71\xf5\xed\x1d\xe4\x72\x27\xe3\xf6\xbe\xce\x26\xb2\x5f\x53\xcf\x32\x35\xcb\x95\xe6\xab\xcf\xa8\x25\x8d\x9e\xca\xc7\x91\xc5\x6a\x70\x88\x81\xa9\xbb\xd5\x36\x79\x0c\x85\xf5\xe6\x8a\x75\x28\x57\x1a\xbd\x94\x75\xc2\x5f\x39\x3d\x47\xf3\x61\xd0\x3c\x6c\x76\xdc\x82\xfc\xb0\xd4\xe8\xb8\x3b\x39\x96\x1d\xdf\xd6\x54\x09\x30\x7d\xb3\x6d\x6b\xde\x37\xd7\x0b\x9a\x89\x5c\xd7\x5a\xdd\xea\x0d\x89\x1f\xbb\x8b\x38\x3f\xe2\x79\xa6\xbd\xce\xfd\x7b\xb7\x75\x77\xc9\x59\xf5\x3d\x72\x6b\xc2\x72\x3c\x3f\x3d\x69\x40\xee\x5f\x68\x3a\x11\xd6\xba\xc1\x86\xa6\xee\x05\xf0\x99\x1c\xba\x66\xe4\x28\x8d\x45\x1b\xf1\xbf\xc9\x64\xad\xd5\x72\x56\x24\x67\xac\x62\x26\x64\x3c\xba\x6b\x32\xd0\xde\x1f\x9b\x3d\xf9\x2b\x86\xa5\x67\xe9\xee\x18\xac\x0e\x55\x06\x0e\xdf\xa6\x97\xc2\x89\xfd\xaf\xa3\xc0\x22\x54\x81\x42\x7d\x95\x99\x93\xe5\x8c\x30\x16\x92\xe8\xee\x67\xa5\x89\xd4\x5f\x48\x82\x3f\x3f\x3c\x04\xbb\x1b\x56\x77\x23\xfd\x06\xcc\xb3\xda\xf9\xdc\x3e\x6a\x1d\xc7\xec\x53\x77\xd5\x6b\xbf\x96\xf7\xbe\xbf\xd8\x7f\x49\x32\x9f\x58\x92\x8d\xbb\x3f\x31\x76\xea\x57\x35\xc5\xa4\xfa\xd5\xbe\xbb\xd1\x1f\x8f\xdc\xff\xbb\xfc\x37\x00\x00\xff\xff\x40\x21\x01\x4e\x52\x25\x00\x00") +var _cmdInternalPagesAssetsHtmlContainersHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x5a\x5f\x73\xe2\x38\x12\x7f\x86\x4f\xd1\xeb\xba\x87\xd9\xaa\xb1\x49\x26\xb9\x87\x9b\x23\x54\xb1\xcc\xcc\x2d\xb7\x99\x24\x15\x92\xdd\xda\x47\xd9\x6e\x6c\x4d\x84\xa5\x95\x64\x08\x97\xca\x77\xbf\x92\x64\x83\xff\x01\xf9\x57\x33\xcb\x0b\x20\xa9\xbb\x7f\xfd\x53\xb7\xd4\xb2\x3c\xfc\xc9\xf7\xfb\x00\x13\x2e\xd6\x92\x26\xa9\x86\x0f\x47\xc7\xa7\xf0\x1f\xce\x13\x86\x30\xcd\xa2\x00\xc6\x8c\xc1\xb5\xe9\x52\x70\x8d\x0a\xe5\x12\xe3\xa0\xdf\x07\x38\xa7\x11\x66\x0a\x63\xc8\xb3\x18\x25\xe8\x14\x61\x2c\x48\x94\x62\xd9\xf3\x1e\x7e\x47\xa9\x28\xcf\xe0\x43\x70\x04\xef\xcc\x00\xaf\xe8\xf2\x7e\xfe\x77\x1f\x60\xcd\x73\x58\x90\x35\x64\x5c\x43\xae\x10\x74\x4a\x15\xcc\x29\x43\xc0\xfb\x08\x85\x06\x9a\x41\xc4\x17\x82\x51\x92\x45\x08\x2b\xaa\x53\x6b\xa6\x50\x12\xf4\x01\xfe\x2c\x54\xf0\x50\x13\x9a\x01\x81\x88\x8b\x35\xf0\x79\x75\x1c\x10\x6d\xf0\x9a\x4f\xaa\xb5\xf8\x38\x18\xac\x56\xab\x80\x58\xac\x01\x97\xc9\x80\xb9\x71\x6a\x70\x3e\x9d\x7c\xbe\x98\x7d\xf6\x3f\x04\x47\x46\xe2\x36\x63\xa8\x14\x48\xfc\x2b\xa7\x12\x63\x08\xd7\x40\x84\x60\x34\x22\x21\x43\x60\x64\x05\x5c\x02\x49\x24\x62\x0c\x9a\x1b\xb4\x2b\x49\x35\xcd\x92\xf7\xa0\xf8\x5c\xaf\x88\xc4\x3e\x40\x4c\x95\x96\x34\xcc\x75\x8d\xaa\x12\x1b\x55\xb5\x01\x3c\x03\x92\x81\x37\x9e\xc1\x74\xe6\xc1\x2f\xe3\xd9\x74\xf6\xbe\x0f\xf0\xc7\xf4\xe6\xd7\xcb\xdb\x1b\xf8\x63\x7c\x7d\x3d\xbe\xb8\x99\x7e\x9e\xc1\xe5\x35\x4c\x2e\x2f\x3e\x4d\x6f\xa6\x97\x17\x33\xb8\xfc\x02\xe3\x8b\x3f\xe1\xb7\xe9\xc5\xa7\xf7\x80\x54\xa7\x28\x01\xef\x85\x34\xf8\xb9\x04\x6a\x48\x34\xf3\x06\x30\x43\xac\x01\x98\x73\x07\x48\x09\x8c\xe8\x9c\x46\xc0\x48\x96\xe4\x24\x41\x48\xf8\x12\x65\x46\xb3\x04\x04\xca\x05\x55\x66\x2a\x15\x90\x2c\xee\x03\x30\xba\xa0\x9a\x68\xdb\xd2\x72\x2a\xe8\xfb\xfe\xa8\xdf\x1f\xa6\x7a\xc1\x46\x7d\x80\x61\x8a\x24\x1e\xd9\x29\x18\x6a\xaa\x19\x8e\xa2\x71\xbc\xa4\x8a\x4b\xf0\xe1\xe1\x21\xf8\x44\x95\x60\x64\x7d\x41\x16\xf8\xf8\x38\x1c\xb8\x21\x6e\xb8\x8a\x24\x15\x1a\x94\x8c\xce\xbc\x87\x87\xe0\x9a\x73\xfd\xf8\xa8\x8c\xe5\x68\x20\xb8\x10\x28\x83\x05\xcd\x82\x6f\xca\x1b\x0d\x07\x6e\x70\x21\xf9\x93\xef\xc3\x39\xd1\xa8\xb4\x8d\x21\xca\x30\x36\xd8\x61\x41\x33\x3a\xa7\x18\xc3\x64\x36\x03\x83\xd3\x8e\x66\x34\xbb\x03\x89\xec\xcc\x53\x7a\xcd\x50\xa5\x88\xda\x83\x54\xe2\xbc\x6d\x37\xe4\x5c\x2b\x2d\x89\xf0\x4f\x83\xa3\xe0\xc8\x0f\x51\x93\xe0\x83\xc5\x11\x29\xe5\x8d\xfa\x5b\x00\x97\xc2\x50\x44\x98\x61\x67\x81\xaf\x35\x67\x95\xf8\x27\xc1\x71\x70\xdc\xb2\xf6\x1c\x8d\x11\xcf\x4c\xb6\xa0\x54\x2d\xc0\x7b\x19\xfb\x2f\x59\x92\x99\x9b\x90\x8d\x27\xfb\x26\xe8\xdb\x5f\x39\xca\xb5\x7f\x12\xfc\xb3\x00\xdc\x31\x4d\xfb\xe4\xf7\x10\xdd\xd6\xb4\xd5\xa5\xd7\x02\xcf\x3c\x8d\xf7\x7a\xf0\x8d\x2c\x89\x6b\xf5\xba\x4d\x30\x4e\x62\x94\x7b\x80\x3d\x47\x59\x85\xd7\xa6\xc2\xe1\xa0\xcc\x81\x61\xc8\xe3\x75\x61\x23\xa6\x4b\x88\x18\x51\xea\xcc\xdb\xc8\xba\x50\xf1\x55\xca\x57\x11\x51\xe8\xc1\xc6\x3d\xd2\x9c\x4e\x6f\x2b\xcc\x7c\xb5\xf0\x8f\x3f\x78\x40\xe3\x33\x8f\xf1\x84\x7b\x1b\xb1\x01\xd9\xfc\xac\xd9\x2b\x45\x46\xfd\x5e\xb5\x43\x90\x04\x7d\x03\x16\xa5\xe9\x32\xd9\x7b\x3c\x6a\x27\x69\x7a\x6c\xe4\x06\x31\x5d\x9a\x6f\xce\x4a\xf1\x50\x22\x89\x23\x99\x2f\x42\x27\xfd\xf0\x20\x49\x96\x20\xfc\x43\x10\x89\x99\x9e\x6c\xdc\xfc\x78\x06\xc1\x55\xbd\x4d\x3d\x3e\x5a\x83\x8c\x8e\x2a\xce\x36\x25\x83\x73\x9a\xdd\x3d\x3e\x7a\xa3\x8e\xae\x1b\xbc\xd7\x06\x1d\x19\x0d\x07\x8c\x16\x00\x30\x8b\x8d\xe2\xe1\x80\xb3\x2d\x29\x16\xb8\xfb\xf3\xf0\x40\xe7\x10\x4c\x95\x23\xf5\x00\x57\x50\x7c\x86\xe9\xe9\x16\x64\x10\x0c\x62\x1e\xdd\x19\xc6\x3e\xd9\x6f\xd8\xfa\xe4\xc0\xa4\xa7\x9d\xa6\x0f\x59\x69\xdb\x11\x3c\x5e\x90\xcc\x1b\x5d\xd9\xef\xa7\xda\x29\x49\xa8\x3a\x3c\xcb\xc3\xa8\xca\xfc\xeb\x62\xe4\x64\x54\xd3\x37\x1c\xa4\x27\xd5\x00\xa9\x08\x33\xaa\xb4\x9f\x48\x9e\x8b\x46\x84\xa8\x8a\x02\x1b\x1e\x4d\x84\xbd\x5a\x12\xd4\xc6\x97\x41\xd1\x36\xe2\x53\x8d\x0b\x1b\x2c\xb5\xf1\xdb\x48\x69\x04\x49\x75\x76\x76\x52\xe8\x18\x74\x73\x3d\xd3\x44\xe7\x6f\x41\xe0\x27\x49\x97\x28\xc1\xe9\x6b\x12\x98\xb3\x83\xfc\xb9\x10\x54\x56\xdc\xf2\xd7\xc0\xe7\x52\xcb\xa9\x81\x0e\x8a\x86\x4a\x90\xac\xb4\x62\xd4\xf8\x8c\x84\xc8\x2c\x77\x55\xdd\xc1\x6f\xb8\x36\xd4\x99\xe1\x23\x68\x76\xfe\x4e\x58\x6e\x57\x88\x66\xfe\xd5\x59\x73\xce\x6e\xb1\xf5\x5e\x06\x6d\xa6\xb9\x24\x09\x0e\x43\x39\x2a\x00\x19\x55\xbb\xc8\xea\x6d\xb9\xb2\xe6\x5b\x5c\xed\x46\xf5\x5c\xbe\x2a\xfa\xdb\x7c\x55\x3b\xeb\x7c\xf5\x36\x74\xf5\x86\x83\x9c\x59\x6f\x4a\x26\x8b\x86\x5d\xd1\xda\x95\xe3\xce\xab\xe9\x82\x24\x78\x38\x42\x2b\x8b\xce\xce\x50\x05\xa8\x2e\x4d\x27\x23\xa7\xda\x05\x6b\xa5\xa7\x8a\xcb\x69\x33\xfb\x92\x8b\x13\x9f\x5a\x19\xb3\x3f\xd6\x46\x99\x29\x0c\xe5\xf6\xff\x21\xdf\xae\x51\xf1\x5c\x46\xa8\xc6\x4b\x42\x99\x29\xc9\xdf\x20\x07\xa7\x8a\x33\x5b\xd6\x36\xf2\xcf\x99\x9c\x88\xbc\x6a\x6c\x67\xa0\x55\x98\xd8\x19\x3f\x40\x22\x4d\x97\xe6\x00\x50\x58\xf4\x6d\xdd\x0b\x82\x64\xc8\xdc\x6f\x6f\x34\xb9\xba\x75\xd3\xbf\xd5\x58\x2c\xde\x02\x23\x03\x27\x38\x37\x85\xf8\xc6\xf1\xfd\x26\xf7\xe5\x51\x4a\xa4\x99\xc7\x32\x46\x85\xa4\x99\x76\x8d\x6d\x63\x50\x53\x93\x67\x74\xa3\x46\x55\xd5\xb4\x91\x57\x27\xb1\xc3\x97\xaf\xe4\xfe\x8d\xdc\xf9\x4a\xee\xc1\xaa\x6a\x78\x34\xe1\x75\x87\xb6\x16\x77\xfb\x14\xf1\x57\xb9\xa4\xee\x5e\xef\xce\x98\x31\xbe\x32\x47\x16\xde\x9e\x24\x63\xa1\x61\x10\x82\xaf\x24\x4a\x69\x86\xd3\x6c\xce\x83\x8b\x7c\x61\xe5\xca\x35\xa6\x8d\xbe\x5c\x6a\x36\xff\x9d\x13\x5f\x71\xc1\xe5\xfa\xfb\x06\xbc\xb3\xb9\x27\xe6\xdd\x80\xc0\x3d\x89\xb0\x6a\x5e\x4f\x6f\x45\x59\x33\x03\xe8\xff\x70\x8f\xe1\xdd\x41\x53\xc8\xdf\x66\x54\xef\x91\x7f\x49\x54\x15\x7a\xde\x28\x51\xba\x92\xa4\xed\xf4\xc1\x1c\xd9\xe9\x6e\x21\xf9\x0a\x47\x67\x2b\x22\xde\x6a\x91\x5b\x11\xd1\xb9\x2c\xb4\x3d\xae\x58\x7d\x81\xd7\x15\xe9\x03\x9e\x37\x53\xaf\xf0\xee\x29\x67\x84\xc3\x9b\xd9\xad\x32\xa5\xd1\xee\x4a\xdc\x66\x5e\x91\x7f\x42\xd2\x05\x91\xeb\x3d\x65\x80\x19\x65\x2c\xd0\x2c\x69\x17\x02\xf5\x61\x45\x32\x5f\x2e\x51\x2e\x29\xae\xf6\x97\x07\xd5\x0a\x21\x37\x88\xfd\x84\xe4\x09\x7a\x75\x95\xe6\xd4\xbc\x29\x19\x7e\x88\x37\x57\x92\x47\xa8\xd4\xa1\x6a\xa7\xea\x8e\x28\x45\x7c\xcd\xc5\x93\x1c\xda\x51\x67\x7c\x47\x37\x6d\xc9\xf1\x14\x07\x3b\xbc\x69\x18\x38\x1d\xdd\x70\x4d\x18\x94\x71\x78\x6a\x23\xb3\xc2\x4f\x24\x72\x5f\x9b\x21\xbe\x9b\xf8\x28\x25\x52\x6f\x49\x81\xf2\xa9\x94\x51\x35\xb9\xba\x85\x73\x4e\x62\x18\x2f\x51\xee\xd1\xc7\x38\x89\xeb\x8a\x36\x0f\xab\xaa\xc8\x2c\x26\x10\xf6\xa8\x2e\x77\x2a\x13\x28\x7d\xb3\xff\x77\xe2\xeb\x56\xf9\x8b\x44\x72\x17\xf3\x55\xb6\x4b\xa7\x53\x15\x96\xc3\x76\x2a\x6d\x87\xc6\xc1\xdd\xf9\x3b\x86\x49\xb9\x51\x7f\xa7\x48\x59\x58\x73\x87\xa7\x21\x94\x83\x46\x4b\x05\x80\xe4\x2b\xe8\x3e\xf0\xec\x9d\xc2\xc6\xb0\xf6\x72\xfc\x2f\x7b\xb6\xac\xb9\x2a\x79\x22\xd1\x3e\x5b\x85\xd6\xa7\x6b\xa0\x1f\x12\x09\xd5\x3f\x7e\x6c\x0e\xaa\xd2\x2b\xd7\x11\xd7\x91\x72\xed\x3b\x2a\x3a\x35\x43\x7d\xaf\x52\xd2\xe7\x19\x5b\x7b\xa3\x5f\xb9\x86\x72\xc2\xdc\x21\xb9\x43\xb2\xcd\xe6\x73\xe0\xd2\x6c\xce\x1b\x60\x23\xce\xe2\x97\xa0\x9d\x70\x16\x3f\x15\x6e\xaf\xd7\x89\xbb\xbb\xb1\x3d\x73\x27\x5e\x35\xba\x34\xde\x37\x57\x9f\x67\x26\xe5\x05\xea\x15\x97\x77\xcf\xcc\xca\xde\xeb\xd3\xb1\x30\x5c\x6c\xf6\xcf\x49\xc4\x5e\xb3\x37\x96\x5c\x98\xe0\x6f\x27\x48\x98\x6b\xcd\x37\xf3\x15\xea\x0c\x42\x9d\xf9\x31\xce\x49\xce\x34\x94\x72\xbe\xe6\x49\xc2\xd0\x2b\x1e\x9d\x3b\x21\xc7\x73\xe6\x50\xfa\x0a\x19\x46\xf6\x08\xb0\x31\x06\x31\xd1\xa4\x10\xad\x60\x00\x22\x29\xf1\x53\xa2\x04\x17\xb9\x38\xf3\xb4\xcc\xb1\x68\xc4\x7b\x41\xb2\x18\xe3\x33\x6f\x4e\x98\xc2\x8e\x10\x73\xe1\xd5\x6d\xb8\x9c\xeb\xee\xf8\xaa\x05\x66\x44\x24\x56\xc6\xf6\xca\x48\x70\x9e\xb5\x58\xca\x59\xb7\x49\xaf\x49\xb0\xbf\xc0\x2c\xf7\x40\x72\xe3\xb1\xfb\x6d\x1d\xb3\xd5\x25\xc3\x38\x5c\xef\x65\xac\x1d\xf3\xc5\xe3\xa1\x3d\x61\xfb\x9c\x05\x39\x95\x3c\x4f\x52\x91\xeb\xf6\x2a\xb8\x59\x96\x4b\x78\xe1\x5a\xa3\x6a\x6f\xdf\x2f\x30\xfb\x59\x4a\x6e\x1f\x1f\xb7\xb6\x80\xd2\x16\xda\x11\xbb\x8d\x35\x9c\x6f\x64\xe8\x17\xf5\xc3\xb6\xcc\x2f\x94\xa1\x5a\x2b\x8d\x8b\xa7\x57\x90\xf3\x8d\x8c\xdb\xfb\x3a\x8b\xc8\xdd\x9a\x76\x2c\x53\x93\x5c\x69\xbe\xf8\x8a\x5a\xd2\xe8\xb9\x7c\x1c\x58\xac\x7a\xfb\x18\x18\xbb\xdb\x73\x13\xc7\x50\x58\x6f\xae\x58\xfb\x62\xa5\x51\x4b\x59\x27\xfc\x85\xd3\x73\x30\x1e\x7a\xcd\xc3\x66\xc7\x2d\xc8\x0f\x0b\x8d\x8e\xbb\x93\x43\xd1\xf1\xb4\xa2\x4a\x80\xa9\x9b\x6d\x59\xf3\xb1\xb9\x5e\xd0\x4c\xe4\xba\x56\xea\x56\x6f\x48\xfc\xd8\x5d\xf8\xf9\x11\xcf\x33\xed\x75\xee\xdf\x9b\xad\xbb\x4b\xce\xaa\xdf\x21\xb7\x24\x2c\xc7\xb3\xe3\xa3\x06\xe4\xdd\x0b\x4d\x27\xc2\x5a\x35\xd8\xd0\xd4\xbd\x00\xbe\x90\x43\x57\x8c\x1c\xa4\xb1\x28\x23\xfe\x9e\x4c\xd6\x4a\x2d\x67\x45\x72\xc6\x2a\x66\x42\xc6\xa3\x3b\xaf\xab\x7e\xde\xe7\xdc\xcb\x27\x61\xc7\x42\xdd\xd1\x59\xed\xaa\x74\xec\xbf\xa3\x2f\x85\x13\xfb\x2e\x53\x60\x11\xaa\x40\xa1\xbe\xcc\xcc\x39\x72\x42\x18\x0b\x49\x74\xf7\x4e\x69\x22\xf5\x15\x49\xf0\xdd\xc3\x43\xb0\xb9\x4f\x75\xf7\xdc\xef\xc1\xb4\xd5\x4e\xe3\xb6\xa9\x75\xf8\xb2\xad\xee\x02\xd9\xfe\x2c\x6f\x93\x7f\xb6\x2f\x3a\x99\x4f\x2c\xc9\xca\xdd\x96\x18\x3b\xf5\x8b\x99\x62\x50\xfd\x85\x01\xf7\x9e\xc0\x70\xe0\xde\xa2\xf9\x7f\x00\x00\x00\xff\xff\x19\x81\xf2\xde\xa8\x25\x00\x00") func cmdInternalPagesAssetsHtmlContainersHtmlBytes() ([]byte, error) { return bindataRead( @@ -99,7 +99,7 @@ func cmdInternalPagesAssetsHtmlContainersHtml() (*asset, error) { } info := bindataFileInfo{name: "cmd/internal/pages/assets/html/containers.html", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2e, 0xea, 0xa6, 0x77, 0xb6, 0x4d, 0x35, 0xb4, 0x31, 0xcb, 0x2a, 0x5, 0xb4, 0x7c, 0xcb, 0x7e, 0xcf, 0x72, 0xd, 0xe7, 0x7e, 0xd0, 0x68, 0x82, 0x40, 0xa7, 0x5f, 0xd7, 0xf3, 0x45, 0x2a, 0xc1}} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x3b, 0xb9, 0x8c, 0xe6, 0xbf, 0xf1, 0x9d, 0x7, 0xe7, 0xbc, 0xd5, 0xf1, 0xb1, 0xc2, 0x47, 0xe5, 0x2e, 0x76, 0xdf, 0x59, 0x12, 0x64, 0x50, 0x9d, 0x91, 0xb3, 0xaf, 0xb3, 0x1, 0x60, 0xd, 0x6c}} return a, nil } diff --git a/container/container.go b/container/container.go index 8414efc6a0..4c435a0e81 100644 --- a/container/container.go +++ b/container/container.go @@ -35,6 +35,7 @@ const ( ContainerTypeCrio ContainerTypeContainerd ContainerTypeMesos + ContainerTypePodman ) // Interface for container operation handlers. diff --git a/container/docker/docker.go b/container/docker/docker.go index 05934bf970..4c01c370d7 100644 --- a/container/docker/docker.go +++ b/container/docker/docker.go @@ -19,12 +19,12 @@ import ( "fmt" "regexp" "strconv" + "time" dockertypes "github.com/docker/docker/api/types" "golang.org/x/net/context" - "time" - + "github.com/google/cadvisor/container/docker/utils" v1 "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/machine" ) @@ -88,63 +88,55 @@ func Images() ([]v1.DockerImage, error) { if err != nil { return nil, fmt.Errorf("unable to communicate with docker daemon: %v", err) } - images, err := client.ImageList(defaultContext(), dockertypes.ImageListOptions{All: false}) + summaries, err := client.ImageList(defaultContext(), dockertypes.ImageListOptions{All: false}) if err != nil { return nil, err } - - out := []v1.DockerImage{} - const unknownTag = ":" - for _, image := range images { - if len(image.RepoTags) == 1 && image.RepoTags[0] == unknownTag { - // images with repo or tags are uninteresting. - continue - } - di := v1.DockerImage{ - ID: image.ID, - RepoTags: image.RepoTags, - Created: image.Created, - VirtualSize: image.VirtualSize, - Size: image.Size, - } - out = append(out, di) - } - return out, nil - + return utils.SummariesToImages(summaries) } // Checks whether the dockerInfo reflects a valid docker setup, and returns it if it does, or an // error otherwise. -func ValidateInfo() (*dockertypes.Info, error) { - client, err := Client() - if err != nil { - return nil, fmt.Errorf("unable to communicate with docker daemon: %v", err) - } - - dockerInfo, err := client.Info(defaultContext()) +func ValidateInfo(GetInfo func() (*dockertypes.Info, error), ServerVersion func() (string, error)) (*dockertypes.Info, error) { + info, err := GetInfo() if err != nil { - return nil, fmt.Errorf("failed to detect Docker info: %v", err) + return nil, err } // Fall back to version API if ServerVersion is not set in info. - if dockerInfo.ServerVersion == "" { - version, err := client.ServerVersion(defaultContext()) + if info.ServerVersion == "" { + var err error + info.ServerVersion, err = ServerVersion() if err != nil { - return nil, fmt.Errorf("unable to get docker version: %v", err) + return nil, fmt.Errorf("unable to get runtime version: %v", err) } - dockerInfo.ServerVersion = version.Version } - version, err := parseVersion(dockerInfo.ServerVersion, versionRe, 3) + + version, err := ParseVersion(info.ServerVersion, VersionRe, 3) if err != nil { return nil, err } if version[0] < 1 { - return nil, fmt.Errorf("cAdvisor requires docker version %v or above but we have found version %v reported as %q", []int{1, 0, 0}, version, dockerInfo.ServerVersion) + return nil, fmt.Errorf("cAdvisor requires runtime version %v or above but we have found version %v reported as %q", []int{1, 0, 0}, version, info.ServerVersion) + } + + if info.Driver == "" { + return nil, fmt.Errorf("failed to find runtime storage driver") } - if dockerInfo.Driver == "" { - return nil, fmt.Errorf("failed to find docker storage driver") + return info, nil +} + +func Info() (*dockertypes.Info, error) { + client, err := Client() + if err != nil { + return nil, fmt.Errorf("unable to communicate with docker daemon: %v", err) + } + + dockerInfo, err := client.Info(defaultContext()) + if err != nil { + return nil, fmt.Errorf("failed to detect Docker info: %v", err) } return &dockerInfo, nil @@ -155,7 +147,7 @@ func APIVersion() ([]int, error) { if err != nil { return nil, err } - return parseVersion(ver, apiVersionRe, 2) + return ParseVersion(ver, apiVersionRe, 2) } func VersionString() (string, error) { @@ -182,7 +174,7 @@ func APIVersionString() (string, error) { return apiVersion, err } -func parseVersion(versionString string, regex *regexp.Regexp, length int) ([]int, error) { +func ParseVersion(versionString string, regex *regexp.Regexp, length int) ([]int, error) { matches := regex.FindAllStringSubmatch(versionString, -1) if len(matches) != 1 { return nil, fmt.Errorf("version string \"%v\" doesn't match expected regular expression: \"%v\"", versionString, regex.String()) diff --git a/container/docker/docker_test.go b/container/docker/docker_test.go index 0fed09ef17..5d31764c5f 100644 --- a/container/docker/docker_test.go +++ b/container/docker/docker_test.go @@ -28,14 +28,14 @@ func TestParseDockerAPIVersion(t *testing.T) { expected []int expectedError string }{ - {"17.03.0", versionRe, 3, []int{17, 03, 0}, ""}, - {"17.a3.0", versionRe, 3, []int{}, `version string "17.a3.0" doesn't match expected regular expression: "(\d+)\.(\d+)\.(\d+)"`}, + {"17.03.0", VersionRe, 3, []int{17, 03, 0}, ""}, + {"17.a3.0", VersionRe, 3, []int{}, `version string "17.a3.0" doesn't match expected regular expression: "(\d+)\.(\d+)\.(\d+)"`}, {"1.20", apiVersionRe, 2, []int{1, 20}, ""}, {"1.a", apiVersionRe, 2, []int{}, `version string "1.a" doesn't match expected regular expression: "(\d+)\.(\d+)"`}, } for _, test := range tests { - actual, err := parseVersion(test.version, test.regex, test.length) + actual, err := ParseVersion(test.version, test.regex, test.length) if err != nil { if len(test.expectedError) == 0 { t.Errorf("%s: expected no error, got %v", test.version, err) diff --git a/container/docker/factory.go b/container/docker/factory.go index 3f2e442f76..d9a371616d 100644 --- a/container/docker/factory.go +++ b/container/docker/factory.go @@ -17,7 +17,6 @@ package docker import ( "flag" "fmt" - "path" "regexp" "strconv" "strings" @@ -59,10 +58,6 @@ const rootDirRetries = 5 // The retry period for getting docker root dir, Millisecond const rootDirRetryPeriod time.Duration = 1000 * time.Millisecond -// Regexp that identifies docker cgroups, containers started with -// --cgroup-parent have another prefix than 'docker' -var dockerCgroupRegexp = regexp.MustCompile(`([a-z0-9]{64})`) - var ( // Basepath to all container specific information that libcontainer stores. dockerRootDir string @@ -96,21 +91,21 @@ func RootDir() string { return dockerRootDir } -type storageDriver string +type StorageDriver string const ( - devicemapperStorageDriver storageDriver = "devicemapper" - aufsStorageDriver storageDriver = "aufs" - overlayStorageDriver storageDriver = "overlay" - overlay2StorageDriver storageDriver = "overlay2" - zfsStorageDriver storageDriver = "zfs" - vfsStorageDriver storageDriver = "vfs" + DevicemapperStorageDriver StorageDriver = "devicemapper" + AufsStorageDriver StorageDriver = "aufs" + OverlayStorageDriver StorageDriver = "overlay" + Overlay2StorageDriver StorageDriver = "overlay2" + ZfsStorageDriver StorageDriver = "zfs" + VfsStorageDriver StorageDriver = "vfs" ) type dockerFactory struct { machineInfoFactory info.MachineInfoFactory - storageDriver storageDriver + storageDriver StorageDriver storageDir string client *docker.Client @@ -169,36 +164,15 @@ func (f *dockerFactory) NewContainerHandler(name string, metadataEnvAllowList [] return } -// Returns the Docker ID from the full container name. -func ContainerNameToDockerId(name string) string { - id := path.Base(name) - - if matches := dockerCgroupRegexp.FindStringSubmatch(id); matches != nil { - return matches[1] - } - - return id -} - -// isContainerName returns true if the cgroup with associated name -// corresponds to a docker container. -func isContainerName(name string) bool { - // always ignore .mount cgroup even if associated with docker and delegate to systemd - if strings.HasSuffix(name, ".mount") { - return false - } - return dockerCgroupRegexp.MatchString(path.Base(name)) -} - // Docker handles all containers under /docker func (f *dockerFactory) CanHandleAndAccept(name string) (bool, bool, error) { // if the container is not associated with docker, we can't handle it or accept it. - if !isContainerName(name) { + if !dockerutil.IsContainerName(name) { return false, false, nil } // Check if the container is known to docker and it is active. - id := ContainerNameToDockerId(name) + id := dockerutil.ContainerNameToId(name) // We assume that if Inspect fails then the container is not known to docker. ctnr, err := f.client.ContainerInspect(context.Background(), id) @@ -215,12 +189,12 @@ func (f *dockerFactory) DebugInfo() map[string][]string { var ( versionRegexpString = `(\d+)\.(\d+)\.(\d+)` - versionRe = regexp.MustCompile(versionRegexpString) + VersionRe = regexp.MustCompile(versionRegexpString) apiVersionRegexpString = `(\d+)\.(\d+)` apiVersionRe = regexp.MustCompile(apiVersionRegexpString) ) -func startThinPoolWatcher(dockerInfo *dockertypes.Info) (*devicemapper.ThinPoolWatcher, error) { +func StartThinPoolWatcher(dockerInfo *dockertypes.Info) (*devicemapper.ThinPoolWatcher, error) { _, err := devicemapper.ThinLsBinaryPresent() if err != nil { return nil, err @@ -253,7 +227,7 @@ func startThinPoolWatcher(dockerInfo *dockertypes.Info) (*devicemapper.ThinPoolW return thinPoolWatcher, nil } -func startZfsWatcher(dockerInfo *dockertypes.Info) (*zfs.ZfsWatcher, error) { +func StartZfsWatcher(dockerInfo *dockertypes.Info) (*zfs.ZfsWatcher, error) { filesystem, err := dockerutil.DockerZfsFilesystem(*dockerInfo) if err != nil { return nil, err @@ -275,7 +249,7 @@ func ensureThinLsKernelVersion(kernelVersion string) error { // thin_ls to work without corrupting the thin pool minRhel7KernelVersion := semver.MustParse("3.10.0") - matches := versionRe.FindStringSubmatch(kernelVersion) + matches := VersionRe.FindStringSubmatch(kernelVersion) if len(matches) < 4 { return fmt.Errorf("error parsing kernel version: %q is not a semver", kernelVersion) } @@ -335,13 +309,13 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics return fmt.Errorf("unable to communicate with docker daemon: %v", err) } - dockerInfo, err := ValidateInfo() + dockerInfo, err := ValidateInfo(Info, VersionString) if err != nil { return fmt.Errorf("failed to validate Docker info: %v", err) } // Version already validated above, assume no error here. - dockerVersion, _ := parseVersion(dockerInfo.ServerVersion, versionRe, 3) + dockerVersion, _ := ParseVersion(dockerInfo.ServerVersion, VersionRe, 3) dockerAPIVersion, _ := APIVersion() @@ -356,8 +330,8 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics zfsWatcher *zfs.ZfsWatcher ) if includedMetrics.Has(container.DiskUsageMetrics) { - if storageDriver(dockerInfo.Driver) == devicemapperStorageDriver { - thinPoolWatcher, err = startThinPoolWatcher(dockerInfo) + if StorageDriver(dockerInfo.Driver) == DevicemapperStorageDriver { + thinPoolWatcher, err = StartThinPoolWatcher(dockerInfo) if err != nil { klog.Errorf("devicemapper filesystem stats will not be reported: %v", err) } @@ -367,8 +341,8 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics thinPoolName = status.DriverStatus[dockerutil.DriverStatusPoolName] } - if storageDriver(dockerInfo.Driver) == zfsStorageDriver { - zfsWatcher, err = startZfsWatcher(dockerInfo) + if StorageDriver(dockerInfo.Driver) == ZfsStorageDriver { + zfsWatcher, err = StartZfsWatcher(dockerInfo) if err != nil { klog.Errorf("zfs filesystem stats will not be reported: %v", err) } @@ -383,7 +357,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics dockerAPIVersion: dockerAPIVersion, fsInfo: fsInfo, machineInfoFactory: factory, - storageDriver: storageDriver(dockerInfo.Driver), + storageDriver: StorageDriver(dockerInfo.Driver), storageDir: RootDir(), includedMetrics: includedMetrics, thinPoolName: thinPoolName, diff --git a/container/docker/factory_test.go b/container/docker/factory_test.go index 17e960fd10..3131aac858 100644 --- a/container/docker/factory_test.go +++ b/container/docker/factory_test.go @@ -49,24 +49,3 @@ func TestEnsureThinLsKernelVersion(t *testing.T) { } } } - -func TestIsContainerName(t *testing.T) { - tests := []struct { - name string - expected bool - }{ - { - name: "/system.slice/var-lib-docker-overlay-9f086b233ab7c786bf8b40b164680b658a8f00e94323868e288d6ce20bc92193-merged.mount", - expected: false, - }, - { - name: "/system.slice/docker-72e5a5ff5eef3c4222a6551b992b9360a99122f77d2229783f0ee0946dfd800e.scope", - expected: true, - }, - } - for _, test := range tests { - if actual := isContainerName(test.name); actual != test.expected { - t.Errorf("%s: expected: %v, actual: %v", test.name, test.expected, actual) - } - } -} diff --git a/container/docker/fs.go b/container/docker/fs.go new file mode 100644 index 0000000000..79384d0e40 --- /dev/null +++ b/container/docker/fs.go @@ -0,0 +1,173 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "fmt" + + "k8s.io/klog/v2" + + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/common" + "github.com/google/cadvisor/devicemapper" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/zfs" +) + +func FsStats( + stats *info.ContainerStats, + machineInfoFactory info.MachineInfoFactory, + metrics container.MetricSet, + storageDriver StorageDriver, + fsHandler common.FsHandler, + globalFsInfo fs.FsInfo, + poolName string, + rootfsStorageDir string, + zfsParent string, +) error { + mi, err := machineInfoFactory.GetMachineInfo() + if err != nil { + return err + } + + if metrics.Has(container.DiskIOMetrics) { + common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo) + } + + if metrics.Has(container.DiskUsageMetrics) { + var device string + switch storageDriver { + case DevicemapperStorageDriver: + device = poolName + case AufsStorageDriver, OverlayStorageDriver, Overlay2StorageDriver, VfsStorageDriver: + deviceInfo, err := globalFsInfo.GetDirFsDevice(rootfsStorageDir) + if err != nil { + return fmt.Errorf("unable to determine device info for dir: %v: %v", rootfsStorageDir, err) + } + device = deviceInfo.Device + case ZfsStorageDriver: + device = zfsParent + default: + return nil + } + + for _, fs := range mi.Filesystems { + if fs.Device == device { + usage := fsHandler.Usage() + fsStat := info.FsStats{ + Device: device, + Type: fs.Type, + Limit: fs.Capacity, + BaseUsage: usage.BaseUsageBytes, + Usage: usage.TotalUsageBytes, + Inodes: usage.InodeUsage, + } + fileSystems, err := globalFsInfo.GetGlobalFsInfo() + if err != nil { + return fmt.Errorf("unable to obtain diskstats for filesystem %s: %v", fsStat.Device, err) + } + addDiskStats(fileSystems, &fs, &fsStat) + stats.Filesystem = append(stats.Filesystem, fsStat) + break + } + } + } + + return nil +} + +func addDiskStats(fileSystems []fs.Fs, fsInfo *info.FsInfo, fsStats *info.FsStats) { + if fsInfo == nil { + return + } + + for _, fileSys := range fileSystems { + if fsInfo.DeviceMajor == fileSys.DiskStats.Major && + fsInfo.DeviceMinor == fileSys.DiskStats.Minor { + fsStats.ReadsCompleted = fileSys.DiskStats.ReadsCompleted + fsStats.ReadsMerged = fileSys.DiskStats.ReadsMerged + fsStats.SectorsRead = fileSys.DiskStats.SectorsRead + fsStats.ReadTime = fileSys.DiskStats.ReadTime + fsStats.WritesCompleted = fileSys.DiskStats.WritesCompleted + fsStats.WritesMerged = fileSys.DiskStats.WritesMerged + fsStats.SectorsWritten = fileSys.DiskStats.SectorsWritten + fsStats.WriteTime = fileSys.DiskStats.WriteTime + fsStats.IoInProgress = fileSys.DiskStats.IoInProgress + fsStats.IoTime = fileSys.DiskStats.IoTime + fsStats.WeightedIoTime = fileSys.DiskStats.WeightedIoTime + break + } + } +} + +// FsHandler is a composite FsHandler implementation the incorporates +// the common fs handler, a devicemapper ThinPoolWatcher, and a zfsWatcher +type FsHandler struct { + FsHandler common.FsHandler + + // thinPoolWatcher is the devicemapper thin pool watcher + ThinPoolWatcher *devicemapper.ThinPoolWatcher + // deviceID is the id of the container's fs device + DeviceID string + + // zfsWatcher is the zfs filesystem watcher + ZfsWatcher *zfs.ZfsWatcher + // zfsFilesystem is the docker zfs filesystem + ZfsFilesystem string +} + +var _ common.FsHandler = &FsHandler{} + +func (h *FsHandler) Start() { + h.FsHandler.Start() +} + +func (h *FsHandler) Stop() { + h.FsHandler.Stop() +} + +func (h *FsHandler) Usage() common.FsUsage { + usage := h.FsHandler.Usage() + + // When devicemapper is the storage driver, the base usage of the container comes from the thin pool. + // We still need the result of the fsHandler for any extra storage associated with the container. + // To correctly factor in the thin pool usage, we should: + // * Usage the thin pool usage as the base usage + // * Calculate the overall usage by adding the overall usage from the fs handler to the thin pool usage + if h.ThinPoolWatcher != nil { + thinPoolUsage, err := h.ThinPoolWatcher.GetUsage(h.DeviceID) + if err != nil { + // TODO: ideally we should keep track of how many times we failed to get the usage for this + // device vs how many refreshes of the cache there have been, and display an error e.g. if we've + // had at least 1 refresh and we still can't find the device. + klog.V(5).Infof("unable to get fs usage from thin pool for device %s: %v", h.DeviceID, err) + } else { + usage.BaseUsageBytes = thinPoolUsage + usage.TotalUsageBytes += thinPoolUsage + } + } + + if h.ZfsWatcher != nil { + zfsUsage, err := h.ZfsWatcher.GetUsage(h.ZfsFilesystem) + if err != nil { + klog.V(5).Infof("unable to get fs usage from zfs for filesystem %s: %v", h.ZfsFilesystem, err) + } else { + usage.BaseUsageBytes = zfsUsage + usage.TotalUsageBytes += zfsUsage + } + } + return usage +} diff --git a/container/docker/handler.go b/container/docker/handler.go index 710ebb5d03..fc66641f6f 100644 --- a/container/docker/handler.go +++ b/container/docker/handler.go @@ -35,7 +35,6 @@ import ( docker "github.com/docker/docker/client" "golang.org/x/net/context" - "k8s.io/klog/v2" ) const ( @@ -57,7 +56,7 @@ type dockerContainerHandler struct { cgroupPaths map[string]string // the docker storage driver - storageDriver storageDriver + storageDriver StorageDriver fsInfo fs.FsInfo rootfsStorageDir string @@ -93,7 +92,7 @@ type dockerContainerHandler struct { var _ container.ContainerHandler = &dockerContainerHandler{} -func getRwLayerID(containerID, storageDir string, sd storageDriver, dockerVersion []int) (string, error) { +func getRwLayerID(containerID, storageDir string, sd StorageDriver, dockerVersion []int) (string, error) { const ( // Docker version >=1.10.0 have a randomized ID for the root fs of a container. randomizedRWLayerMinorVersion = 10 @@ -116,7 +115,7 @@ func newDockerContainerHandler( name string, machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, - storageDriver storageDriver, + storageDriver StorageDriver, storageDir string, cgroupSubsystems map[string]string, inHostNamespace bool, @@ -142,7 +141,7 @@ func newDockerContainerHandler( storageDir = path.Join(rootFs, storageDir) } - id := ContainerNameToDockerId(name) + id := dockerutil.ContainerNameToId(name) // Add the Containers dir where the log files are stored. // FIXME: Give `otherStorageDir` a more descriptive name. @@ -155,27 +154,9 @@ func newDockerContainerHandler( // Determine the rootfs storage dir OR the pool name to determine the device. // For devicemapper, we only need the thin pool name, and that is passed in to this call - var ( - rootfsStorageDir string - zfsFilesystem string - zfsParent string - ) - switch storageDriver { - case aufsStorageDriver: - rootfsStorageDir = path.Join(storageDir, string(aufsStorageDriver), aufsRWLayer, rwLayerID) - case overlayStorageDriver: - rootfsStorageDir = path.Join(storageDir, string(storageDriver), rwLayerID, overlayRWLayer) - case overlay2StorageDriver: - rootfsStorageDir = path.Join(storageDir, string(storageDriver), rwLayerID, overlay2RWLayer) - case vfsStorageDriver: - rootfsStorageDir = path.Join(storageDir) - case zfsStorageDriver: - status, err := Status() - if err != nil { - return nil, fmt.Errorf("unable to determine docker status: %v", err) - } - zfsParent = status.DriverStatus[dockerutil.DriverStatusParentDataset] - zfsFilesystem = path.Join(zfsParent, rwLayerID) + rootfsStorageDir, zfsFilesystem, zfsParent, err := DetermineDeviceStorage(storageDriver, storageDir, rwLayerID) + if err != nil { + return nil, fmt.Errorf("unable to determine device storage: %v", err) } // We assume that if Inspect fails then the container is not known to docker. @@ -238,12 +219,12 @@ func newDockerContainerHandler( handler.ipAddress = ipAddress if includedMetrics.Has(container.DiskUsageMetrics) { - handler.fsHandler = &dockerFsHandler{ - fsHandler: common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo), - thinPoolWatcher: thinPoolWatcher, - zfsWatcher: zfsWatcher, - deviceID: ctnr.GraphDriver.Data["DeviceId"], - zfsFilesystem: zfsFilesystem, + handler.fsHandler = &FsHandler{ + FsHandler: common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo), + ThinPoolWatcher: thinPoolWatcher, + ZfsWatcher: zfsWatcher, + DeviceID: ctnr.GraphDriver.Data["DeviceId"], + ZfsFilesystem: zfsFilesystem, } } @@ -267,63 +248,27 @@ func newDockerContainerHandler( return handler, nil } -// dockerFsHandler is a composite FsHandler implementation the incorporates -// the common fs handler, a devicemapper ThinPoolWatcher, and a zfsWatcher -type dockerFsHandler struct { - fsHandler common.FsHandler - - // thinPoolWatcher is the devicemapper thin pool watcher - thinPoolWatcher *devicemapper.ThinPoolWatcher - // deviceID is the id of the container's fs device - deviceID string - - // zfsWatcher is the zfs filesystem watcher - zfsWatcher *zfs.ZfsWatcher - // zfsFilesystem is the docker zfs filesystem - zfsFilesystem string -} - -var _ common.FsHandler = &dockerFsHandler{} - -func (h *dockerFsHandler) Start() { - h.fsHandler.Start() -} - -func (h *dockerFsHandler) Stop() { - h.fsHandler.Stop() -} - -func (h *dockerFsHandler) Usage() common.FsUsage { - usage := h.fsHandler.Usage() - - // When devicemapper is the storage driver, the base usage of the container comes from the thin pool. - // We still need the result of the fsHandler for any extra storage associated with the container. - // To correctly factor in the thin pool usage, we should: - // * Usage the thin pool usage as the base usage - // * Calculate the overall usage by adding the overall usage from the fs handler to the thin pool usage - if h.thinPoolWatcher != nil { - thinPoolUsage, err := h.thinPoolWatcher.GetUsage(h.deviceID) - if err != nil { - // TODO: ideally we should keep track of how many times we failed to get the usage for this - // device vs how many refreshes of the cache there have been, and display an error e.g. if we've - // had at least 1 refresh and we still can't find the device. - klog.V(5).Infof("unable to get fs usage from thin pool for device %s: %v", h.deviceID, err) - } else { - usage.BaseUsageBytes = thinPoolUsage - usage.TotalUsageBytes += thinPoolUsage - } - } - - if h.zfsWatcher != nil { - zfsUsage, err := h.zfsWatcher.GetUsage(h.zfsFilesystem) +func DetermineDeviceStorage(storageDriver StorageDriver, storageDir string, rwLayerID string) ( + rootfsStorageDir string, zfsFilesystem string, zfsParent string, err error) { + switch storageDriver { + case AufsStorageDriver: + rootfsStorageDir = path.Join(storageDir, string(AufsStorageDriver), aufsRWLayer, rwLayerID) + case OverlayStorageDriver: + rootfsStorageDir = path.Join(storageDir, string(storageDriver), rwLayerID, overlayRWLayer) + case Overlay2StorageDriver: + rootfsStorageDir = path.Join(storageDir, string(storageDriver), rwLayerID, overlay2RWLayer) + case VfsStorageDriver: + rootfsStorageDir = path.Join(storageDir) + case ZfsStorageDriver: + var status info.DockerStatus + status, err = Status() if err != nil { - klog.V(5).Infof("unable to get fs usage from zfs for filesystem %s: %v", h.zfsFilesystem, err) - } else { - usage.BaseUsageBytes = zfsUsage - usage.TotalUsageBytes += zfsUsage + return } + zfsParent = status.DriverStatus[dockerutil.DriverStatusParentDataset] + zfsFilesystem = path.Join(zfsParent, rwLayerID) } - return usage + return } func (h *dockerContainerHandler) Start() { @@ -355,99 +300,6 @@ func (h *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) { return spec, err } -func (h *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error { - mi, err := h.machineInfoFactory.GetMachineInfo() - if err != nil { - return err - } - - if h.includedMetrics.Has(container.DiskIOMetrics) { - common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo) - } - - if !h.includedMetrics.Has(container.DiskUsageMetrics) { - return nil - } - var device string - switch h.storageDriver { - case devicemapperStorageDriver: - // Device has to be the pool name to correlate with the device name as - // set in the machine info filesystems. - device = h.poolName - case aufsStorageDriver, overlayStorageDriver, overlay2StorageDriver, vfsStorageDriver: - deviceInfo, err := h.fsInfo.GetDirFsDevice(h.rootfsStorageDir) - if err != nil { - return fmt.Errorf("unable to determine device info for dir: %v: %v", h.rootfsStorageDir, err) - } - device = deviceInfo.Device - case zfsStorageDriver: - device = h.zfsParent - default: - return nil - } - - var ( - limit uint64 - fsType string - ) - - var fsInfo *info.FsInfo - - // Docker does not impose any filesystem limits for containers. So use capacity as limit. - for _, fs := range mi.Filesystems { - if fs.Device == device { - limit = fs.Capacity - fsType = fs.Type - fsInfo = &fs - break - } - } - - fsStat := info.FsStats{Device: device, Type: fsType, Limit: limit} - usage := h.fsHandler.Usage() - fsStat.BaseUsage = usage.BaseUsageBytes - fsStat.Usage = usage.TotalUsageBytes - fsStat.Inodes = usage.InodeUsage - - if fsInfo != nil { - fileSystems, err := h.fsInfo.GetGlobalFsInfo() - - if err == nil { - addDiskStats(fileSystems, fsInfo, &fsStat) - } else { - klog.Errorf("Unable to obtain diskstats for filesystem %s: %v", fsStat.Device, err) - } - } - - stats.Filesystem = append(stats.Filesystem, fsStat) - - return nil -} - -func addDiskStats(fileSystems []fs.Fs, fsInfo *info.FsInfo, fsStats *info.FsStats) { - if fsInfo == nil { - return - } - - for _, fileSys := range fileSystems { - if fsInfo.DeviceMajor == fileSys.DiskStats.Major && - fsInfo.DeviceMinor == fileSys.DiskStats.Minor { - fsStats.ReadsCompleted = fileSys.DiskStats.ReadsCompleted - fsStats.ReadsMerged = fileSys.DiskStats.ReadsMerged - fsStats.SectorsRead = fileSys.DiskStats.SectorsRead - fsStats.ReadTime = fileSys.DiskStats.ReadTime - fsStats.WritesCompleted = fileSys.DiskStats.WritesCompleted - fsStats.WritesMerged = fileSys.DiskStats.WritesMerged - fsStats.SectorsWritten = fileSys.DiskStats.SectorsWritten - fsStats.WriteTime = fileSys.DiskStats.WriteTime - fsStats.IoInProgress = fileSys.DiskStats.IoInProgress - fsStats.IoTime = fileSys.DiskStats.IoTime - fsStats.WeightedIoTime = fileSys.DiskStats.WeightedIoTime - break - } - } -} - // TODO(vmarmol): Get from libcontainer API instead of cgroup manager when we don't have to support older Dockers. func (h *dockerContainerHandler) GetStats() (*info.ContainerStats, error) { stats, err := h.libcontainerHandler.GetStats() @@ -456,7 +308,8 @@ func (h *dockerContainerHandler) GetStats() (*info.ContainerStats, error) { } // Get filesystem stats. - err = h.getFsStats(stats) + err = FsStats(stats, h.machineInfoFactory, h.includedMetrics, h.storageDriver, + h.fsHandler, h.fsInfo, h.poolName, h.rootfsStorageDir, h.zfsParent) if err != nil { return stats, err } diff --git a/container/docker/handler_test.go b/container/docker/handler_test.go index b1cec70aea..e65acc9f6f 100644 --- a/container/docker/handler_test.go +++ b/container/docker/handler_test.go @@ -30,7 +30,7 @@ import ( func TestStorageDirDetectionWithOldVersions(t *testing.T) { as := assert.New(t) - rwLayer, err := getRwLayerID("abcd", "/", aufsStorageDriver, []int{1, 9, 0}) + rwLayer, err := getRwLayerID("abcd", "/", AufsStorageDriver, []int{1, 9, 0}) as.Nil(err) as.Equal(rwLayer, "abcd") } diff --git a/container/docker/utils/docker.go b/container/docker/utils/docker.go index 658607f736..11a0c9e9f1 100644 --- a/container/docker/utils/docker.go +++ b/container/docker/utils/docker.go @@ -17,9 +17,12 @@ package utils import ( "fmt" "os" + "path" + "regexp" "strings" dockertypes "github.com/docker/docker/api/types" + v1 "github.com/google/cadvisor/info/v1" ) const ( @@ -28,6 +31,10 @@ const ( DriverStatusParentDataset = "Parent Dataset" ) +// Regexp that identifies docker cgroups, containers started with +// --cgroup-parent have another prefix than 'docker' +var cgroupRegexp = regexp.MustCompile(`([a-z0-9]{64})`) + func DriverStatusValue(status [][2]string, target string) string { for _, v := range status { if strings.EqualFold(v[0], target) { @@ -75,3 +82,44 @@ func DockerZfsFilesystem(info dockertypes.Info) (string, error) { return filesystem, nil } + +func SummariesToImages(summaries []dockertypes.ImageSummary) ([]v1.DockerImage, error) { + var out []v1.DockerImage + const unknownTag = ":" + for _, summary := range summaries { + if len(summary.RepoTags) == 1 && summary.RepoTags[0] == unknownTag { + // images with repo or tags are uninteresting. + continue + } + di := v1.DockerImage{ + ID: summary.ID, + RepoTags: summary.RepoTags, + Created: summary.Created, + VirtualSize: summary.VirtualSize, + Size: summary.Size, + } + out = append(out, di) + } + return out, nil +} + +// Returns the ID from the full container name. +func ContainerNameToId(name string) string { + id := path.Base(name) + + if matches := cgroupRegexp.FindStringSubmatch(id); matches != nil { + return matches[1] + } + + return id +} + +// IsContainerName returns true if the cgroup with associated name +// corresponds to a container. +func IsContainerName(name string) bool { + // always ignore .mount cgroup even if associated with docker and delegate to systemd + if strings.HasSuffix(name, ".mount") { + return false + } + return cgroupRegexp.MatchString(path.Base(name)) +} diff --git a/container/docker/utils/docker_test.go b/container/docker/utils/docker_test.go new file mode 100644 index 0000000000..9f2b13ec2a --- /dev/null +++ b/container/docker/utils/docker_test.go @@ -0,0 +1,38 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import "testing" + +func TestIsContainerName(t *testing.T) { + tests := []struct { + name string + expected bool + }{ + { + name: "/system.slice/var-lib-docker-overlay-9f086b233ab7c786bf8b40b164680b658a8f00e94323868e288d6ce20bc92193-merged.mount", + expected: false, + }, + { + name: "/system.slice/docker-72e5a5ff5eef3c4222a6551b992b9360a99122f77d2229783f0ee0946dfd800e.scope", + expected: true, + }, + } + for _, test := range tests { + if actual := IsContainerName(test.name); actual != test.expected { + t.Errorf("%s: expected: %v, actual: %v", test.name, test.expected, actual) + } + } +} diff --git a/container/podman/client.go b/container/podman/client.go new file mode 100644 index 0000000000..af33ebeaf5 --- /dev/null +++ b/container/podman/client.go @@ -0,0 +1,58 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podman + +import ( + "context" + "fmt" + "net" + "net/http" + urllib "net/url" +) + +type clientKey struct{} + +func (c clientKey) String() string { + return "client" +} + +type Connection struct { + URI *urllib.URL + Client *http.Client +} + +func client(ctx *context.Context) (*Connection, error) { + url, err := urllib.Parse(*endpointFlag) + if err != nil { + return nil, err + } + + switch url.Scheme { + case "unix": + connection := Connection{URI: url} + connection.Client = &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", url.Path) + }, + DisableCompression: true, + }, + } + *ctx = context.WithValue(*ctx, clientKey{}, &connection) + return &connection, nil + } + + return nil, fmt.Errorf("couldn't get podman client") +} diff --git a/container/podman/factory.go b/container/podman/factory.go new file mode 100644 index 0000000000..ed0cd0b987 --- /dev/null +++ b/container/podman/factory.go @@ -0,0 +1,113 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podman + +import ( + "flag" + "fmt" + "path" + "sync" + "time" + + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/docker" + dockerutil "github.com/google/cadvisor/container/docker/utils" + "github.com/google/cadvisor/devicemapper" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/zfs" +) + +const ( + rootDirRetries = 5 + rootDirRetryPeriod = time.Second + containerBaseName = "container" +) + +var ( + endpointFlag = flag.String("podman", "unix:///var/run/podman/podman.sock", "podman endpoint") +) + +var ( + rootDir string + rootDirOnce sync.Once +) + +func RootDir() string { + rootDirOnce.Do(func() { + for i := 0; i < rootDirRetries; i++ { + status, err := Status() + if err == nil && status.RootDir != "" { + rootDir = status.RootDir + break + } else { + time.Sleep(rootDirRetryPeriod) + } + } + }) + return rootDir +} + +type podmanFactory struct { + // Information about the mounted cgroup subsystems. + machineInfoFactory info.MachineInfoFactory + + storageDriver docker.StorageDriver + storageDir string + + cgroupSubsystem map[string]string + + fsInfo fs.FsInfo + + metrics container.MetricSet + + thinPoolName string + thinPoolWatcher *devicemapper.ThinPoolWatcher + + zfsWatcher *zfs.ZfsWatcher +} + +func (f *podmanFactory) CanHandleAndAccept(name string) (handle bool, accept bool, err error) { + // Rootless + if path.Base(name) == containerBaseName { + name, _ = path.Split(name) + } + if !dockerutil.IsContainerName(name) { + return false, false, nil + } + + id := dockerutil.ContainerNameToId(name) + + ctnr, err := InspectContainer(id) + if err != nil || !ctnr.State.Running { + return false, true, fmt.Errorf("error inspecting container: %v", err) + } + + return true, true, nil +} + +func (f *podmanFactory) DebugInfo() map[string][]string { + return map[string][]string{} +} + +func (f *podmanFactory) String() string { + return "podman" +} + +func (f *podmanFactory) NewContainerHandler(name string, metadataEnvAllowList []string, inHostNamespace bool) (handler container.ContainerHandler, err error) { + return newPodmanContainerHandler(name, f.machineInfoFactory, f.fsInfo, + f.storageDriver, f.storageDir, f.cgroupSubsystem, inHostNamespace, + metadataEnvAllowList, f.metrics, f.thinPoolName, f.thinPoolWatcher, f.zfsWatcher) +} diff --git a/container/podman/fs.go b/container/podman/fs.go new file mode 100644 index 0000000000..e714e900c3 --- /dev/null +++ b/container/podman/fs.go @@ -0,0 +1,54 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podman + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/google/cadvisor/container/docker" +) + +const ( + containersJSONFilename = "containers.json" +) + +type containersJSON struct { + ID string `json:"id"` + Layer string `json:"layer"` + // rest in unnecessary +} + +func rwLayerID(storageDriver docker.StorageDriver, storageDir string, containerID string) (string, error) { + data, err := os.ReadFile(filepath.Join(storageDir, string(storageDriver)+"-containers", containersJSONFilename)) + if err != nil { + return "", err + } + var containers []containersJSON + err = json.Unmarshal(data, &containers) + if err != nil { + return "", err + } + + for _, c := range containers { + if c.ID == containerID { + return c.Layer, nil + } + } + + return "", fmt.Errorf("unable to determine %v rw layer id", containerID) +} diff --git a/container/podman/handler.go b/container/podman/handler.go new file mode 100644 index 0000000000..203d9b4eb9 --- /dev/null +++ b/container/podman/handler.go @@ -0,0 +1,309 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podman + +import ( + "fmt" + "path" + "path/filepath" + "strings" + "time" + + dockercontainer "github.com/docker/docker/api/types/container" + "github.com/opencontainers/runc/libcontainer/cgroups" + + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/common" + "github.com/google/cadvisor/container/docker" + dockerutil "github.com/google/cadvisor/container/docker/utils" + containerlibcontainer "github.com/google/cadvisor/container/libcontainer" + "github.com/google/cadvisor/devicemapper" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/zfs" +) + +type podmanContainerHandler struct { + // machineInfoFactory provides info.MachineInfo + machineInfoFactory info.MachineInfoFactory + + // Absolute path to the cgroup hierarchies of this container. + // (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test") + cgroupPaths map[string]string + + storageDriver docker.StorageDriver + fsInfo fs.FsInfo + rootfsStorageDir string + + creationTime time.Time + + // Metadata associated with the container. + envs map[string]string + labels map[string]string + + image string + + networkMode dockercontainer.NetworkMode + + fsHandler common.FsHandler + + ipAddress string + + metrics container.MetricSet + + thinPoolName string + + zfsParent string + + reference info.ContainerReference + + libcontainerHandler *containerlibcontainer.Handler +} + +func newPodmanContainerHandler( + name string, + machineInfoFactory info.MachineInfoFactory, + fsInfo fs.FsInfo, + storageDriver docker.StorageDriver, + storageDir string, + cgroupSubsystems map[string]string, + inHostNamespace bool, + metadataEnvAllowList []string, + metrics container.MetricSet, + thinPoolName string, + thinPoolWatcher *devicemapper.ThinPoolWatcher, + zfsWatcher *zfs.ZfsWatcher, +) (container.ContainerHandler, error) { + // Create the cgroup paths. + cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems, name) + + cgroupManager, err := containerlibcontainer.NewCgroupManager(name, cgroupPaths) + if err != nil { + return nil, err + } + + rootFs := "/" + if !inHostNamespace { + rootFs = "/rootfs" + storageDir = path.Join(rootFs, storageDir) + } + + rootless := path.Base(name) == containerBaseName + if rootless { + name, _ = path.Split(name) + } + + id := dockerutil.ContainerNameToId(name) + + // We assume that if Inspect fails then the container is not known to Podman. + ctnr, err := InspectContainer(id) + if err != nil { + return nil, err + } + + rwLayerID, err := rwLayerID(storageDriver, storageDir, id) + if err != nil { + return nil, err + } + + rootfsStorageDir, zfsParent, zfsFilesystem, err := determineDeviceStorage(storageDriver, storageDir, rwLayerID) + if err != nil { + return nil, err + } + + otherStorageDir := filepath.Join(storageDir, string(storageDriver)+"-containers", id) + + handler := &podmanContainerHandler{ + machineInfoFactory: machineInfoFactory, + cgroupPaths: cgroupPaths, + storageDriver: storageDriver, + fsInfo: fsInfo, + rootfsStorageDir: rootfsStorageDir, + ipAddress: ctnr.NetworkSettings.IPAddress, + envs: make(map[string]string), + labels: ctnr.Config.Labels, + image: ctnr.Config.Image, + networkMode: ctnr.HostConfig.NetworkMode, + fsHandler: common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo), + metrics: metrics, + thinPoolName: thinPoolName, + zfsParent: zfsParent, + reference: info.ContainerReference{ + Id: id, + Name: name, + Aliases: []string{strings.TrimPrefix(ctnr.Name, "/"), id}, + Namespace: Namespace, + }, + libcontainerHandler: containerlibcontainer.NewHandler(cgroupManager, rootFs, ctnr.State.Pid, metrics), + } + + handler.creationTime, err = time.Parse(time.RFC3339, ctnr.Created) + if err != nil { + return nil, fmt.Errorf("failed to parse the create timestamp %q for container %q: %v", ctnr.Created, id, err) + } + + if ctnr.RestartCount > 0 { + handler.labels["restartcount"] = fmt.Sprint(ctnr.RestartCount) + } + + // Obtain the IP address for the container. + // If the NetworkMode starts with 'container:' then we need to use the IP address of the container specified. + // This happens in cases such as kubernetes where the containers doesn't have an IP address itself and we need to use the pod's address + networkMode := string(handler.networkMode) + if handler.ipAddress == "" && strings.HasPrefix(networkMode, "container:") { + id := strings.TrimPrefix(networkMode, "container:") + ctnr, err := InspectContainer(id) + if err != nil { + return nil, err + } + handler.ipAddress = ctnr.NetworkSettings.IPAddress + } + + if metrics.Has(container.DiskUsageMetrics) { + handler.fsHandler = &docker.FsHandler{ + FsHandler: common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo), + ThinPoolWatcher: thinPoolWatcher, + ZfsWatcher: zfsWatcher, + DeviceID: ctnr.GraphDriver.Data["DeviceId"], + ZfsFilesystem: zfsFilesystem, + } + } + + // Split env vars to get metadata map. + for _, exposedEnv := range metadataEnvAllowList { + if exposedEnv == "" { + continue + } + + for _, envVar := range ctnr.Config.Env { + if envVar != "" { + splits := strings.SplitN(envVar, "=", 2) + if len(splits) == 2 && strings.HasPrefix(splits[0], exposedEnv) { + handler.envs[strings.ToLower(splits[0])] = splits[1] + } + } + } + } + + return handler, nil +} + +func determineDeviceStorage(storageDriver docker.StorageDriver, storageDir string, rwLayerID string) ( + rootfsStorageDir string, zfsFilesystem string, zfsParent string, err error) { + switch storageDriver { + // Podman aliased the driver names together. + case docker.OverlayStorageDriver, docker.Overlay2StorageDriver: + rootfsStorageDir = path.Join(storageDir, "overlay", rwLayerID, "diff") + return + default: + return docker.DetermineDeviceStorage(storageDriver, storageDir, rwLayerID) + } +} + +func (p podmanContainerHandler) ContainerReference() (info.ContainerReference, error) { + return p.reference, nil +} + +func (p podmanContainerHandler) needNet() bool { + if p.metrics.Has(container.NetworkUsageMetrics) { + p.networkMode.IsContainer() + return !p.networkMode.IsContainer() + } + return false +} + +func (p podmanContainerHandler) GetSpec() (info.ContainerSpec, error) { + hasFilesystem := p.metrics.Has(container.DiskUsageMetrics) + + spec, err := common.GetSpec(p.cgroupPaths, p.machineInfoFactory, p.needNet(), hasFilesystem) + if err != nil { + return info.ContainerSpec{}, err + } + + spec.Labels = p.labels + spec.Envs = p.envs + spec.Image = p.image + spec.CreationTime = p.creationTime + + return spec, nil +} + +func (p podmanContainerHandler) GetStats() (*info.ContainerStats, error) { + stats, err := p.libcontainerHandler.GetStats() + if err != nil { + return stats, err + } + + if !p.needNet() { + stats.Network = info.NetworkStats{} + } + + err = docker.FsStats(stats, p.machineInfoFactory, p.metrics, p.storageDriver, + p.fsHandler, p.fsInfo, p.thinPoolName, p.rootfsStorageDir, p.zfsParent) + if err != nil { + return stats, err + } + + return stats, nil +} + +func (p podmanContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) { + return []info.ContainerReference{}, nil +} + +func (p podmanContainerHandler) ListProcesses(listType container.ListType) ([]int, error) { + return p.libcontainerHandler.GetProcesses() +} + +func (p podmanContainerHandler) GetCgroupPath(resource string) (string, error) { + var res string + if !cgroups.IsCgroup2UnifiedMode() { + res = resource + } + path, ok := p.cgroupPaths[res] + if !ok { + return "", fmt.Errorf("couldn't find path for resource %q for container %q", resource, p.reference.Name) + } + + return path, nil +} + +func (p podmanContainerHandler) GetContainerLabels() map[string]string { + return p.labels +} + +func (p podmanContainerHandler) GetContainerIPAddress() string { + return p.ipAddress +} + +func (p podmanContainerHandler) Exists() bool { + return common.CgroupExists(p.cgroupPaths) +} + +func (p podmanContainerHandler) Cleanup() { + if p.fsHandler != nil { + p.fsHandler.Stop() + } +} + +func (p podmanContainerHandler) Start() { + if p.fsHandler != nil { + p.fsHandler.Start() + } +} + +func (p podmanContainerHandler) Type() container.ContainerType { + return container.ContainerTypePodman +} diff --git a/container/podman/install/install.go b/container/podman/install/install.go new file mode 100644 index 0000000000..72a273ef02 --- /dev/null +++ b/container/podman/install/install.go @@ -0,0 +1,29 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package install + +import ( + "k8s.io/klog/v2" + + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/podman" +) + +func init() { + err := container.RegisterPlugin("podman", podman.NewPlugin()) + if err != nil { + klog.Fatalf("Failed to register podman plugin: %v", err) + } +} diff --git a/container/podman/plugin.go b/container/podman/plugin.go new file mode 100644 index 0000000000..1aac12b3e8 --- /dev/null +++ b/container/podman/plugin.go @@ -0,0 +1,109 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podman + +import ( + "fmt" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "k8s.io/klog/v2" + + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/docker" + dockerutil "github.com/google/cadvisor/container/docker/utils" + "github.com/google/cadvisor/container/libcontainer" + "github.com/google/cadvisor/devicemapper" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" + "github.com/google/cadvisor/zfs" +) + +func NewPlugin() container.Plugin { + return &plugin{} +} + +type plugin struct{} + +func (p *plugin) InitializeFSContext(context *fs.Context) error { + context.Podman = fs.PodmanContext{ + Root: "", + Driver: "", + DriverStatus: map[string]string{}, + } + + return nil +} + +func (p *plugin) Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) (watcher.ContainerWatcher, error) { + return Register(factory, fsInfo, includedMetrics) +} + +func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, metrics container.MetricSet) (watcher.ContainerWatcher, error) { + cgroupSubsystem, err := libcontainer.GetCgroupSubsystems(metrics) + if err != nil { + return nil, fmt.Errorf("failed to get cgroup subsystems: %v", err) + } + + validatedInfo, err := docker.ValidateInfo(GetInfo, VersionString) + if err != nil { + return nil, fmt.Errorf("failed to validate Podman info: %v", err) + } + + var ( + thinPoolName string + thinPoolWatcher *devicemapper.ThinPoolWatcher + zfsWatcher *zfs.ZfsWatcher + ) + if metrics.Has(container.DiskUsageMetrics) { + switch docker.StorageDriver(validatedInfo.Driver) { + case docker.DevicemapperStorageDriver: + thinPoolWatcher, err = docker.StartThinPoolWatcher(validatedInfo) + if err != nil { + klog.Errorf("devicemapper filesystem stats will not be reported: %v", err) + } + + status, _ := docker.StatusFromDockerInfo(*validatedInfo) + thinPoolName = status.DriverStatus[dockerutil.DriverStatusPoolName] + case docker.ZfsStorageDriver: + zfsWatcher, err = docker.StartZfsWatcher(validatedInfo) + if err != nil { + klog.Errorf("zfs filesystem stats will not be reported: %v", err) + } + } + } + + // Register Podman container handler factory. + klog.V(1).Info("Registering Podman factory") + f := &podmanFactory{ + machineInfoFactory: factory, + storageDriver: docker.StorageDriver(validatedInfo.Driver), + storageDir: RootDir(), + cgroupSubsystem: cgroupSubsystem, + fsInfo: fsInfo, + metrics: metrics, + thinPoolName: thinPoolName, + thinPoolWatcher: thinPoolWatcher, + zfsWatcher: zfsWatcher, + } + + container.RegisterContainerHandlerFactory(f, []watcher.ContainerWatchSource{watcher.Raw}) + + if !cgroups.IsCgroup2UnifiedMode() { + klog.Warning("Podman rootless containers not working with cgroups v1!") + } + + return nil, nil +} diff --git a/container/podman/podman.go b/container/podman/podman.go new file mode 100644 index 0000000000..4f2cc7b95f --- /dev/null +++ b/container/podman/podman.go @@ -0,0 +1,132 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podman + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + dockertypes "github.com/docker/docker/api/types" + "github.com/pkg/errors" + + "github.com/google/cadvisor/container/docker" + "github.com/google/cadvisor/container/docker/utils" + v1 "github.com/google/cadvisor/info/v1" +) + +const ( + Namespace = "podman" +) + +var timeout = 10 * time.Second + +func validateResponse(gotError error, response *http.Response) error { + var err error + switch { + case response == nil: + err = fmt.Errorf("response not present") + case response.StatusCode == http.StatusNotFound: + err = fmt.Errorf("item not found") + case response.StatusCode == http.StatusNotImplemented: + err = fmt.Errorf("query not implemented") + default: + return gotError + } + + if gotError != nil { + err = errors.Wrap(gotError, err.Error()) + } + + return err +} + +func apiGetRequest(url string, item interface{}) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + conn, err := client(&ctx) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + + resp, err := conn.Client.Do(req) + err = validateResponse(err, resp) + if err != nil { + return err + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + + } + + err = json.Unmarshal(data, item) + if err != nil { + return err + } + + return ctx.Err() +} + +func Images() ([]v1.DockerImage, error) { + var summaries []dockertypes.ImageSummary + err := apiGetRequest("http://d/v1.0.0/images/json", &summaries) + if err != nil { + return nil, err + } + return utils.SummariesToImages(summaries) +} + +func Status() (v1.DockerStatus, error) { + podmanInfo, err := GetInfo() + if err != nil { + return v1.DockerStatus{}, err + } + + return docker.StatusFromDockerInfo(*podmanInfo) +} + +func GetInfo() (*dockertypes.Info, error) { + var info dockertypes.Info + err := apiGetRequest("http://d/v1.0.0/info", &info) + return &info, err +} + +func VersionString() (string, error) { + var version dockertypes.Version + err := apiGetRequest("http://d/v1.0.0/version", &version) + if err != nil { + return "Unknown", err + } + + return version.Version, nil +} + +func InspectContainer(id string) (dockertypes.ContainerJSON, error) { + var data dockertypes.ContainerJSON + err := apiGetRequest(fmt.Sprintf("http://d/v1.0.0/containers/%s/json", id), &data) + return data, err +} diff --git a/container/podman/podman_test.go b/container/podman/podman_test.go new file mode 100644 index 0000000000..9414760329 --- /dev/null +++ b/container/podman/podman_test.go @@ -0,0 +1,72 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podman + +import ( + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateResponse(t *testing.T) { + for _, tc := range []struct { + response *http.Response + err error + expected string + }{ + { + response: nil, + err: nil, + expected: "response not present", + }, + { + response: &http.Response{ + StatusCode: http.StatusNotFound, + }, + err: errors.New("some error"), + expected: "item not found: some error", + }, + { + response: &http.Response{ + StatusCode: http.StatusNotImplemented, + }, + err: errors.New("some error"), + expected: "query not implemented: some error", + }, + { + response: &http.Response{ + StatusCode: http.StatusOK, + }, + err: errors.New("some error"), + expected: "some error", + }, + { + response: &http.Response{ + StatusCode: http.StatusOK, + }, + err: nil, + expected: "", + }, + } { + err := validateResponse(tc.err, tc.response) + if tc.expected != "" { + assert.EqualError(t, err, tc.expected) + } else { + assert.NoError(t, err) + } + } +} diff --git a/docs/runtime_options.md b/docs/runtime_options.md index c0b4f89e0c..7307011de7 100644 --- a/docs/runtime_options.md +++ b/docs/runtime_options.md @@ -63,6 +63,12 @@ From [glog](https://github.com/golang/glog) here are some flags we find useful: --docker-tls-ca="ca.pem": trusted CA for TLS-connection with docker ``` +## Podman + +```bash +--podman="unix:///var/run/podman/podman.sock": podman endpoint (default "unix:///var/run/podman/podman.sock") +``` + ## Housekeeping Housekeeping is the periodic actions cAdvisor takes. During these actions, cAdvisor will gather container stats. These flags control how and when cAdvisor performs housekeeping. diff --git a/fs/types.go b/fs/types.go index 35268ace98..269aa8769a 100644 --- a/fs/types.go +++ b/fs/types.go @@ -22,6 +22,7 @@ type Context struct { // docker root directory. Docker DockerContext Crio CrioContext + Podman PodmanContext } type DockerContext struct { @@ -30,6 +31,12 @@ type DockerContext struct { DriverStatus map[string]string } +type PodmanContext struct { + Root string + Driver string + DriverStatus map[string]string +} + type CrioContext struct { Root string } diff --git a/info/v2/container.go b/info/v2/container.go index 15fb79b9ea..f0824027a3 100644 --- a/info/v2/container.go +++ b/info/v2/container.go @@ -25,6 +25,7 @@ import ( const ( TypeName = "name" TypeDocker = "docker" + TypePodman = "podman" ) type CpuSpec struct { diff --git a/manager/manager.go b/manager/manager.go index 8c4396d65d..a23af5a98d 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -30,6 +30,7 @@ import ( "github.com/google/cadvisor/cache/memory" "github.com/google/cadvisor/collector" "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/podman" "github.com/google/cadvisor/container/raw" "github.com/google/cadvisor/events" "github.com/google/cadvisor/fs" @@ -58,8 +59,11 @@ var eventStorageAgeLimit = flag.String("event_storage_age_limit", "default=24h", var eventStorageEventLimit = flag.String("event_storage_event_limit", "default=100000", "Max number of events to store (per type). Value is a comma separated list of key values, where the keys are event types (e.g.: creation, oom) or \"default\" and the value is an integer. Default is applied to all non-specified event types") var applicationMetricsCountLimit = flag.Int("application_metrics_count_limit", 100, "Max number of application metrics to store (per container)") -// The namespace under which Docker aliases are unique. -const DockerNamespace = "docker" +// The namespace under which aliases are unique. +const ( + DockerNamespace = "docker" + PodmanNamespace = "podman" +) var HousekeepingConfigFlags = HouskeepingConfig{ flag.Duration("max_housekeeping_interval", 60*time.Second, "Largest interval to allow between container housekeepings"), @@ -136,6 +140,10 @@ type Manager interface { // Returns debugging information. Map of lines per category. DebugInfo() map[string][]string + + AllPodmanContainers(c *info.ContainerInfoRequest) (map[string]info.ContainerInfo, error) + + PodmanContainer(containerName string, query *info.ContainerInfoRequest) (info.ContainerInfo, error) } // Housekeeping configuration for the manager @@ -265,6 +273,19 @@ type manager struct { containerEnvMetadataWhiteList []string } +func (m *manager) PodmanContainer(containerName string, query *info.ContainerInfoRequest) (info.ContainerInfo, error) { + container, err := m.namespacedContainer(containerName, podman.Namespace) + if err != nil { + return info.ContainerInfo{}, err + } + + inf, err := m.containerDataToContainerInfo(container, query) + if err != nil { + return info.ContainerInfo{}, err + } + return *inf, nil +} + // Start the container manager. func (m *manager) Start() error { m.containerWatchers = container.InitializePlugins(m, m.fsInfo, m.includedMetrics) @@ -581,14 +602,14 @@ func (m *manager) SubcontainersInfo(containerName string, query *info.ContainerI return m.containerDataSliceToContainerInfoSlice(containers, query) } -func (m *manager) getAllDockerContainers() map[string]*containerData { +func (m *manager) getAllNamespacedContainers(ns string) map[string]*containerData { m.containersLock.RLock() defer m.containersLock.RUnlock() containers := make(map[string]*containerData, len(m.containers)) - // Get containers in the Docker namespace. + // Get containers in a namespace. for name, cont := range m.containers { - if name.Namespace == DockerNamespace { + if name.Namespace == ns { containers[cont.info.Name] = cont } } @@ -596,48 +617,34 @@ func (m *manager) getAllDockerContainers() map[string]*containerData { } func (m *manager) AllDockerContainers(query *info.ContainerInfoRequest) (map[string]info.ContainerInfo, error) { - containers := m.getAllDockerContainers() - - output := make(map[string]info.ContainerInfo, len(containers)) - for name, cont := range containers { - inf, err := m.containerDataToContainerInfo(cont, query) - if err != nil { - // Ignore the error because of race condition and return best-effort result. - if err == memory.ErrDataNotFound { - klog.Warningf("Error getting data for container %s because of race condition", name) - continue - } - return nil, err - } - output[name] = *inf - } - return output, nil + containers := m.getAllNamespacedContainers(DockerNamespace) + return m.containersInfo(containers, query) } -func (m *manager) getDockerContainer(containerName string) (*containerData, error) { +func (m *manager) namespacedContainer(containerName string, ns string) (*containerData, error) { m.containersLock.RLock() defer m.containersLock.RUnlock() - // Check for the container in the Docker container namespace. + // Check for the container in the namespace. cont, ok := m.containers[namespacedContainerName{ - Namespace: DockerNamespace, + Namespace: ns, Name: containerName, }] // Look for container by short prefix name if no exact match found. if !ok { for contName, c := range m.containers { - if contName.Namespace == DockerNamespace && strings.HasPrefix(contName.Name, containerName) { + if contName.Namespace == ns && strings.HasPrefix(contName.Name, containerName) { if cont == nil { cont = c } else { - return nil, fmt.Errorf("unable to find container. Container %q is not unique", containerName) + return nil, fmt.Errorf("unable to find container in %q namespace. Container %q is not unique", ns, containerName) } } } if cont == nil { - return nil, fmt.Errorf("unable to find Docker container %q", containerName) + return nil, fmt.Errorf("unable to find container %q in %q namespace", containerName, ns) } } @@ -645,7 +652,7 @@ func (m *manager) getDockerContainer(containerName string) (*containerData, erro } func (m *manager) DockerContainer(containerName string, query *info.ContainerInfoRequest) (info.ContainerInfo, error) { - container, err := m.getDockerContainer(containerName) + container, err := m.namespacedContainer(containerName, DockerNamespace) if err != nil { return info.ContainerInfo{}, err } @@ -717,19 +724,23 @@ func (m *manager) getRequestedContainers(containerName string, options v2.Reques return containersMap, fmt.Errorf("unknown container: %q", containerName) } } - case v2.TypeDocker: + case v2.TypeDocker, v2.TypePodman: + namespace := map[string]string{ + v2.TypeDocker: DockerNamespace, + v2.TypePodman: PodmanNamespace, + }[options.IdType] if !options.Recursive { containerName = strings.TrimPrefix(containerName, "/") - cont, err := m.getDockerContainer(containerName) + cont, err := m.namespacedContainer(containerName, namespace) if err != nil { return containersMap, err } containersMap[cont.info.Name] = cont } else { if containerName != "/" { - return containersMap, fmt.Errorf("invalid request for docker container %q with subcontainers", containerName) + return containersMap, fmt.Errorf("invalid request for %s container %q with subcontainers", options.IdType, containerName) } - containersMap = m.getAllDockerContainers() + containersMap = m.getAllNamespacedContainers(namespace) } default: return containersMap, fmt.Errorf("invalid request type %q", options.IdType) @@ -1357,6 +1368,28 @@ func (m *manager) getFsInfoByDeviceName(deviceName string) (v2.FsInfo, error) { return v2.FsInfo{}, fmt.Errorf("cannot find filesystem info for device %q", deviceName) } +func (m *manager) containersInfo(containers map[string]*containerData, query *info.ContainerInfoRequest) (map[string]info.ContainerInfo, error) { + output := make(map[string]info.ContainerInfo, len(containers)) + for name, cont := range containers { + inf, err := m.containerDataToContainerInfo(cont, query) + if err != nil { + // Ignore the error because of race condition and return best-effort result. + if err == memory.ErrDataNotFound { + klog.Warningf("Error getting data for container %s because of race condition", name) + continue + } + return nil, err + } + output[name] = *inf + } + return output, nil +} + +func (m *manager) AllPodmanContainers(query *info.ContainerInfoRequest) (map[string]info.ContainerInfo, error) { + containers := m.getAllNamespacedContainers(podman.Namespace) + return m.containersInfo(containers, query) +} + func getVersionInfo() (*info.VersionInfo, error) { kernelVersion := machine.KernelVersion() diff --git a/validate/validate.go b/validate/validate.go index 2e953eba3a..e7782d3ff1 100644 --- a/validate/validate.go +++ b/validate/validate.go @@ -207,7 +207,7 @@ func validateCgroups() (string, string) { } func validateDockerInfo() (string, string) { - info, err := docker.ValidateInfo() + info, err := docker.ValidateInfo(docker.Info, docker.VersionString) if err != nil { return Unsupported, fmt.Sprintf("Docker setup is invalid: %v", err) }