Debug k8s failure

Signed-off-by: apostasie <[email protected]>
containerd · Jul 24, 2024 · e0992cf · e0992cf
1 parent fc7da25
commit e0992cf
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 33 deletions.
diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml
@@ -28,7 +28,7 @@ jobs:
       - name: Setup Kind
         uses: engineerd/[email protected]
         with:
-          version: v0.16.0
+          version: v0.23.0
           config: tests/e2e/k8s/kind.yaml
       - name: Build nydus snapshotter dev image
         run: |
@@ -113,13 +113,31 @@ jobs:
 
           sed -e "s|REGISTRY_IP|${registry_ip}|" tests/e2e/k8s/test-pod.yaml.tpl > tests/e2e/k8s/test-pod.yaml
 
-          if [[ "${{ inputs.auth-type }}" == "cri" ]]; then
-            docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet'
-            docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet'
-          fi
+          #if [[ "${{ inputs.auth-type }}" == "cri" ]]; then
+          #  docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet'
+          #fi
+
+          # MountVolume.SetUp failed for volume "kube-api-access-rfppq" : configmap "kube-root-ca.crt" not found
+          # Debug: giving it a restart as the cri auth type exhibits no such issue
+          docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet'
+          docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet'
+          
+          kubectl -n nydus-system get events
+          kubectl -n nydus-system logs nydus-snapshotter
+
+          sleep 2
+          echo "Ready?"
 
           kubectl apply -f tests/e2e/k8s/test-pod.yaml
-          kubectl wait po test-pod -n nydus-system --for=condition=ready --timeout=1m
+          kubectl wait po test-pod -n nydus-system --for=condition=ready --timeout=1m || {
+            kubectl -n nydus-system get events
+            kubectl -n nydus-system logs nydus-snapshotter
+            kubectl -n nydus-system logs test-pod
+            exit 1
+          }
+          # Debug
+          kubectl -n nydus-system get events
+          # Debug
           kubectl delete -f tests/e2e/k8s/test-pod.yaml
       - name: Dump logs
         if: failure()
@@ -149,6 +167,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
+          overwrite: true
           name: k8s-e2e-tests-logs
           path: |
             /tmp/nydus-log
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -75,6 +75,7 @@ jobs:
       - name: upload artifacts
         uses: actions/upload-artifact@v4
         with:
+          overwrite: true
           name: release-tars-${{ matrix.build-os }}-${{ matrix.build-arch }}
           path: |
             package/*.tar.gz*

diff --git a/docs/tarfs.md b/docs/tarfs.md
@@ -24,21 +24,21 @@ $ nerdctl run --snapshotter nydus --rm nginx
 
 # Show mounted rootfs a container
 $ mount
-/dev/loop17 on /var/lib/containerd-nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
 
 # Show loop devices used to mount layers and bootstrap for a container image
 $ losetup 
 NAME SIZELIMIT OFFSET AUTOCLEAR RO BACK-FILE                                                                       DIO LOG-SEC
-/dev/loop11 0      0         0  0 /var/lib/containerd-nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0     512
-/dev/loop12 0      0         0  0 /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0     512
-/dev/loop13 0      0         0  0 /var/lib/containerd-nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0     512
-/dev/loop14 0      0         0  0 /var/lib/containerd-nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0     512
-/dev/loop15 0      0         0  0 /var/lib/containerd-nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0     512
-/dev/loop16 0      0         0  0 /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0     512
-/dev/loop17 0      0         0  0 /var/lib/containerd-nydus/snapshots/7/fs/image/image.boot                         0     512
+/dev/loop11 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0     512
+/dev/loop12 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0     512
+/dev/loop13 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0     512
+/dev/loop14 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0     512
+/dev/loop15 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0     512
+/dev/loop16 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0     512
+/dev/loop17 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/fs/image/image.boot                         0     512
 
 # Files without suffix are tar files, files with suffix `layer.disk` are raw disk image for container image layers
-$ ls -l /var/lib/containerd-nydus/cache/
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
 total 376800
 -rw-r--r-- 1 root root      3584 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
 -rw-r--r-- 1 root root    527872 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk
@@ -54,15 +54,15 @@ total 376800
 -rw-r--r-- 1 root root    529408 Aug 30 23:18 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665.layer.disk
 -rw-r--r-- 1 root root 112968704 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
 -rw-r--r-- 1 root root 113492992 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa.layer.disk
-$ file /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
-/var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive
+$ file /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive
 
 # Mount the raw disk image for a container image layer
-$ losetup /dev/loop100 /var/lib/containerd-nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk 
+$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk 
 $ mount -t erofs /dev/loop100 ./mnt/
 $ mount
 tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1544836k,nr_inodes=386209,mode=700,inode64)
-/dev/loop17 on /var/lib/containerd-nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
 /dev/loop100 on /root/ws/nydus-snapshotter.git/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
 
 ```
@@ -81,7 +81,7 @@ $ containerd-nydus-grpc --config /etc/nydus/config.toml &
 $ nerdctl run --snapshotter nydus --rm nginx
 
 # Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image
-$ ls -l /var/lib/containerd-nydus/cache/
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
 total 376320
 -rw-r--r-- 1 root root      3584 Aug 30 23:35 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
 -rw-r--r-- 1 root root  77814784 Aug 30 23:35 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
@@ -109,7 +109,7 @@ $ containerd-nydus-grpc --config /etc/nydus/config.toml &
 $ nerdctl run --snapshotter nydus --rm nginx
 
 # Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image
-$ ls -l /var/lib/containerd-nydus/cache/
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
 total 388296
 -rw-r--r-- 1 root root      3584 Aug 30 23:45 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
 -rw-r--r-- 1 root root  77814784 Aug 30 23:46 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
@@ -120,7 +120,7 @@ total 388296
 -rw-r--r-- 1 root root      5120 Aug 30 23:45 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665
 -rw-r--r-- 1 root root 112968704 Aug 30 23:46 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
 
-$ losetup /dev/loop100 /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
 $  veritysetup open --no-superblock --format=1 -s "" --hash=sha256 --data-block-size=512 --hash-block-size=4096 --data-blocks 379918 --hash-offset 194519040 /dev/loop100 image1 /dev/loop100 8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd
 $ lsblk
 loop100   7:100  0 197.2M  0 loop  
@@ -136,11 +136,11 @@ $ veritysetup status dm-0
   hash name:   sha256
   salt:        -
   data device: /dev/loop100
-  data loop:   /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+  data loop:   /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
   size:        379918 sectors
   mode:        readonly
   hash device: /dev/loop100
-  hash loop:   /var/lib/containerd-nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+  hash loop:   /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
   hash offset: 379920 sectors
   root hash:   8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd
 

diff --git a/misc/snapshotter/Dockerfile b/misc/snapshotter/Dockerfile
@@ -29,12 +29,13 @@ ARG SCRIPT_DESTINATION=${DESTINATION}/opt/nydus
 WORKDIR /root/
 RUN apk add -q --no-cache libc6-compat bash
 
-VOLUME /var/lib/containerd/io.containerd.snapshotter.v1.nydus /run/containerd-nydus
+VOLUME /var/lib/containerd/io.containerd.snapshotter.v1.nydus
+VOLUME /run/containerd-nydus
 
 COPY --from=sourcer /.nydus_version /.nydus_version
 COPY --from=kubectl-sourcer /usr/bin/kubectl /usr/bin/kubectl
 
-RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd-nydus/cache /tmp/blobs/
+RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache /tmp/blobs/
 COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/
 COPY --chmod=755 containerd-nydus-grpc nydus-overlayfs ${BINARY_DESTINATION}/
 COPY --chmod=755 snapshotter.sh ${SCRIPT_DESTINATION}/snapshotter.sh

diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go
@@ -1033,7 +1033,7 @@ func (o *snapshotter) cleanupDirectories(ctx context.Context) ([]string, error)
 }
 
 func (o *snapshotter) cleanupSnapshotDirectory(ctx context.Context, dir string) error {
-	// For example: cleanupSnapshotDirectory /var/lib/containerd-nydus/snapshots/34" dir=/var/lib/containerd-nydus/snapshots/34
+	// For example: cleanupSnapshotDirectory /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/34" dir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/34
 
 	snapshotID := filepath.Base(dir)
 	if err := o.fs.Umount(ctx, snapshotID); err != nil && !os.IsNotExist(err) {

diff --git a/tests/e2e/k8s/snapshotter-cri.yaml b/tests/e2e/k8s/snapshotter-cri.yaml
@@ -15,7 +15,15 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: nydus-snapshotter-role
 rules:
-  - apiGroups: 
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
     - ""
     resources: 
     - nodes
@@ -168,7 +176,7 @@ data:
   ENABLE_SYSTEMD_SERVICE: "true"
   config.toml: |-
     version = 1
-    root = "/var/lib/containerd-nydus"
+    root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
     address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
     daemon_mode = "multiple"
     enable_system_controller = true

diff --git a/tests/e2e/k8s/snapshotter-kubeconf.yaml b/tests/e2e/k8s/snapshotter-kubeconf.yaml
@@ -23,7 +23,7 @@ rules:
       - get
       - list
       - watch
-  - apiGroups: 
+  - apiGroups:
     - ""
     resources: 
     - nodes
@@ -44,7 +44,6 @@ subjects:
   - kind: ServiceAccount
     name: nydus-snapshotter-sa
     namespace: nydus-system
-
 ---
 apiVersion: v1
 kind: Pod
@@ -174,10 +173,10 @@ data:
   FS_DRIVER: "fusedev"
   ENABLE_CONFIG_FROM_VOLUME: "true"
   ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false"
-  ENABLE_SYSTEMD_SERVICE: "false"
+  ENABLE_SYSTEMD_SERVICE: "true"
   config.toml: |-
     version = 1
-    root = "/var/lib/containerd-nydus"
+    root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
     address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
     daemon_mode = "multiple"
     enable_system_controller = true