From e0992cfee17f7835cee99487c3e292965c2fbb6e Mon Sep 17 00:00:00 2001 From: apostasie Date: Tue, 23 Jul 2024 16:52:04 -0700 Subject: [PATCH] Debug k8s failure Signed-off-by: apostasie --- .github/workflows/k8s-e2e.yml | 31 ++++++++++++++++----- .github/workflows/release.yml | 1 + docs/tarfs.md | 36 ++++++++++++------------- misc/snapshotter/Dockerfile | 5 ++-- snapshot/snapshot.go | 2 +- tests/e2e/k8s/snapshotter-cri.yaml | 12 +++++++-- tests/e2e/k8s/snapshotter-kubeconf.yaml | 7 +++-- 7 files changed, 61 insertions(+), 33 deletions(-) diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index 9296ec7ddc..9199d430d2 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Kind uses: engineerd/setup-kind@v0.5.0 with: - version: v0.16.0 + version: v0.23.0 config: tests/e2e/k8s/kind.yaml - name: Build nydus snapshotter dev image run: | @@ -113,13 +113,31 @@ jobs: sed -e "s|REGISTRY_IP|${registry_ip}|" tests/e2e/k8s/test-pod.yaml.tpl > tests/e2e/k8s/test-pod.yaml - if [[ "${{ inputs.auth-type }}" == "cri" ]]; then - docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet' - docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet' - fi + #if [[ "${{ inputs.auth-type }}" == "cri" ]]; then + # docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet' + #fi + + # MountVolume.SetUp failed for volume "kube-api-access-rfppq" : configmap "kube-root-ca.crt" not found + # Debug: giving it a restart as the cri auth type exhibits no such issue + docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet' + docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet' + + kubectl -n nydus-system get events + kubectl -n nydus-system logs nydus-snapshotter + + sleep 2 + echo "Ready?" kubectl apply -f tests/e2e/k8s/test-pod.yaml - kubectl wait po test-pod -n nydus-system --for=condition=ready --timeout=1m + kubectl wait po test-pod -n nydus-system --for=condition=ready --timeout=1m || { + kubectl -n nydus-system get events + kubectl -n nydus-system logs nydus-snapshotter + kubectl -n nydus-system logs test-pod + exit 1 + } + # Debug + kubectl -n nydus-system get events + # Debug kubectl delete -f tests/e2e/k8s/test-pod.yaml - name: Dump logs if: failure() @@ -149,6 +167,7 @@ jobs: uses: actions/upload-artifact@v4 if: failure() with: + overwrite: true name: k8s-e2e-tests-logs path: | /tmp/nydus-log diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e329d116c..4cd3c31223 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,6 +75,7 @@ jobs: - name: upload artifacts uses: actions/upload-artifact@v4 with: + overwrite: true name: release-tars-${{ matrix.build-os }}-${{ matrix.build-arch }} path: | package/*.tar.gz* diff --git a/docs/tarfs.md b/docs/tarfs.md index da7c0a568f..726305700e 100644 --- a/docs/tarfs.md +++ b/docs/tarfs.md @@ -24,21 +24,21 @@ $ nerdctl run --snapshotter nydus --rm nginx # Show mounted rootfs a container $ mount -/dev/loop17 on /var/lib/containerd-nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) +/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) # Show loop devices used to mount layers and bootstrap for a container image $ losetup NAME SIZELIMIT OFFSET AUTOCLEAR RO BACK-FILE DIO LOG-SEC -/dev/loop11 0 0 0 0 /var/lib/containerd-nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0 512 -/dev/loop12 0 0 0 0 /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0 512 -/dev/loop13 0 0 0 0 /var/lib/containerd-nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0 512 -/dev/loop14 0 0 0 0 /var/lib/containerd-nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0 512 -/dev/loop15 0 0 0 0 /var/lib/containerd-nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0 512 -/dev/loop16 0 0 0 0 /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0 512 -/dev/loop17 0 0 0 0 /var/lib/containerd-nydus/snapshots/7/fs/image/image.boot 0 512 +/dev/loop11 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0 512 +/dev/loop12 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0 512 +/dev/loop13 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0 512 +/dev/loop14 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0 512 +/dev/loop15 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0 512 +/dev/loop16 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0 512 +/dev/loop17 0 0 0 0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/fs/image/image.boot 0 512 # Files without suffix are tar files, files with suffix `layer.disk` are raw disk image for container image layers -$ ls -l /var/lib/containerd-nydus/cache/ +$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/ total 376800 -rw-r--r-- 1 root root 3584 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -rw-r--r-- 1 root root 527872 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk @@ -54,15 +54,15 @@ total 376800 -rw-r--r-- 1 root root 529408 Aug 30 23:18 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665.layer.disk -rw-r--r-- 1 root root 112968704 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa -rw-r--r-- 1 root root 113492992 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa.layer.disk -$ file /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -/var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive +$ file /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 +/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive # Mount the raw disk image for a container image layer -$ losetup /dev/loop100 /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk +$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk $ mount -t erofs /dev/loop100 ./mnt/ $ mount tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1544836k,nr_inodes=386209,mode=700,inode64) -/dev/loop17 on /var/lib/containerd-nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) +/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) /dev/loop100 on /root/ws/nydus-snapshotter.git/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround) ``` @@ -81,7 +81,7 @@ $ containerd-nydus-grpc --config /etc/nydus/config.toml & $ nerdctl run --snapshotter nydus --rm nginx # Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image -$ ls -l /var/lib/containerd-nydus/cache/ +$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/ total 376320 -rw-r--r-- 1 root root 3584 Aug 30 23:35 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -rw-r--r-- 1 root root 77814784 Aug 30 23:35 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5 @@ -109,7 +109,7 @@ $ containerd-nydus-grpc --config /etc/nydus/config.toml & $ nerdctl run --snapshotter nydus --rm nginx # Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image -$ ls -l /var/lib/containerd-nydus/cache/ +$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/ total 388296 -rw-r--r-- 1 root root 3584 Aug 30 23:45 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 -rw-r--r-- 1 root root 77814784 Aug 30 23:46 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5 @@ -120,7 +120,7 @@ total 388296 -rw-r--r-- 1 root root 5120 Aug 30 23:45 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 -rw-r--r-- 1 root root 112968704 Aug 30 23:46 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa -$ losetup /dev/loop100 /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk +$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk $ veritysetup open --no-superblock --format=1 -s "" --hash=sha256 --data-block-size=512 --hash-block-size=4096 --data-blocks 379918 --hash-offset 194519040 /dev/loop100 image1 /dev/loop100 8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd $ lsblk loop100 7:100 0 197.2M 0 loop @@ -136,11 +136,11 @@ $ veritysetup status dm-0 hash name: sha256 salt: - data device: /dev/loop100 - data loop: /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk + data loop: /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk size: 379918 sectors mode: readonly hash device: /dev/loop100 - hash loop: /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk + hash loop: /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk hash offset: 379920 sectors root hash: 8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd diff --git a/misc/snapshotter/Dockerfile b/misc/snapshotter/Dockerfile index c31124270e..041480cd89 100644 --- a/misc/snapshotter/Dockerfile +++ b/misc/snapshotter/Dockerfile @@ -29,12 +29,13 @@ ARG SCRIPT_DESTINATION=${DESTINATION}/opt/nydus WORKDIR /root/ RUN apk add -q --no-cache libc6-compat bash -VOLUME /var/lib/containerd/io.containerd.snapshotter.v1.nydus /run/containerd-nydus +VOLUME /var/lib/containerd/io.containerd.snapshotter.v1.nydus +VOLUME /run/containerd-nydus COPY --from=sourcer /.nydus_version /.nydus_version COPY --from=kubectl-sourcer /usr/bin/kubectl /usr/bin/kubectl -RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd-nydus/cache /tmp/blobs/ +RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache /tmp/blobs/ COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/ COPY --chmod=755 containerd-nydus-grpc nydus-overlayfs ${BINARY_DESTINATION}/ COPY --chmod=755 snapshotter.sh ${SCRIPT_DESTINATION}/snapshotter.sh diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 3b6a61946b..416e1e7854 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -1033,7 +1033,7 @@ func (o *snapshotter) cleanupDirectories(ctx context.Context) ([]string, error) } func (o *snapshotter) cleanupSnapshotDirectory(ctx context.Context, dir string) error { - // For example: cleanupSnapshotDirectory /var/lib/containerd-nydus/snapshots/34" dir=/var/lib/containerd-nydus/snapshots/34 + // For example: cleanupSnapshotDirectory /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/34" dir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/34 snapshotID := filepath.Base(dir) if err := o.fs.Umount(ctx, snapshotID); err != nil && !os.IsNotExist(err) { diff --git a/tests/e2e/k8s/snapshotter-cri.yaml b/tests/e2e/k8s/snapshotter-cri.yaml index 99e87437c3..ff3100b6ca 100644 --- a/tests/e2e/k8s/snapshotter-cri.yaml +++ b/tests/e2e/k8s/snapshotter-cri.yaml @@ -15,7 +15,15 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: nydus-snapshotter-role rules: - - apiGroups: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: - "" resources: - nodes @@ -168,7 +176,7 @@ data: ENABLE_SYSTEMD_SERVICE: "true" config.toml: |- version = 1 - root = "/var/lib/containerd-nydus" + root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus" address = "/run/containerd-nydus/containerd-nydus-grpc.sock" daemon_mode = "multiple" enable_system_controller = true diff --git a/tests/e2e/k8s/snapshotter-kubeconf.yaml b/tests/e2e/k8s/snapshotter-kubeconf.yaml index 3868a9c70f..2fea86d2af 100644 --- a/tests/e2e/k8s/snapshotter-kubeconf.yaml +++ b/tests/e2e/k8s/snapshotter-kubeconf.yaml @@ -23,7 +23,7 @@ rules: - get - list - watch - - apiGroups: + - apiGroups: - "" resources: - nodes @@ -44,7 +44,6 @@ subjects: - kind: ServiceAccount name: nydus-snapshotter-sa namespace: nydus-system - --- apiVersion: v1 kind: Pod @@ -174,10 +173,10 @@ data: FS_DRIVER: "fusedev" ENABLE_CONFIG_FROM_VOLUME: "true" ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false" - ENABLE_SYSTEMD_SERVICE: "false" + ENABLE_SYSTEMD_SERVICE: "true" config.toml: |- version = 1 - root = "/var/lib/containerd-nydus" + root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus" address = "/run/containerd-nydus/containerd-nydus-grpc.sock" daemon_mode = "multiple" enable_system_controller = true