云运维笔记(5) Kubernetes GPU 支持与驱动安装

# 安装lspci命令 

yum install -y pciutils

# 查看 nvidia GPU
lspci | grep -i nvidia

lsmod | grep nouveau
# 如果有结果，则说明存在nouveau。没有则直接跳过这一步。

# 如果出现信息则说明系统自带的 GPU 驱动没有被禁止掉。

# 新建文件
touch /etc/modprobe.d/blacklist-nouveau.conf

# 然后添加如下内容：
blacklist nouveau
options nouveau modeset=0

# 方法一：临时生效，重启无效
modprobe -r nouveau

# 方法二：重启系统
reboot

# 使用安装包卸载
sudo /path/to/NVIDIA-Linux-x86_64-xxx.xx.run --uninstall


# 使用 yum 卸载
sudo yum remove nvidia* -y
sudo yum remove cuda* -y
sudo yum remove "*nvidia*" -y

# 删除文件
sudo rm -rf /usr/local/cuda*
sudo rm -rf /usr/lib/nvidia*
sudo rm -rf /usr/lib64/nvidia*
sudo rm -rf /etc/X11/xorg.conf

# 刷新
sudo yum autoremove
sudo yum autoclean

# 重启机器 
reboot

sh NVIDIA-Linux-x86_64-460.32.03.run -no-x-check -no-nouveau-check -no-opengl-files

ERROR: An NVIDIA kernel module 'nvidia-uvm' appears to already be loaded in your kernel.  This may be because it is in use (for example, by an X server, a CUDA         
         program, or the NVIDIA Persistence Daemon), but this may also happen if your kernel was configured without support for module unloading.  Please be sure to exit 
         any programs that may be using the GPU(s) before attempting to upgrade your driver.  If no GPU-based programs are running, you know that your kernel supports    
         module unloading, and you still receive this message, then an error may have occurred that has corrupted an NVIDIA kernel module's usage count, for which the    
         simplest remedy is to reboot your computer. 

ERROR: Unable to find the kernel source tree for the currently running kernel.  Please make sure you have installed the kernel source files for your kernel and that    
         they are properly configured; on Red Hat Linux systems, for example, be sure you have the 'kernel-source' or 'kernel-devel' RPM installed.  If you know the      
         correct kernel source files are installed, you may specify the kernel source path with the '--kernel-source-path' command line option.     

-> # cat /proc/version    
Linux version 3.10.0-957.el7.x86_64 (mockbuild@kbuilder.bsys.centos.org) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-36) (GCC) ) #1 SMP Thu Nov 8 23:39:32 UTC 2018

-> # rpm -qa |grep kernel
kernel-tools-libs-3.10.0-1160.66.1.el7.x86_64
kernel-tools-3.10.0-1160.66.1.el7.x86_64

# 当前运行是 3.10.0-957.el7 与 rpm 安装列表里的版本不一致。

-> # ll /usr/lib/modules/3.10.0-957.el7.x86_64
total 0
drwxr-xr-x 2 root root 6 Nov 21 10:52 video
-> # ll /usr/src/kernels                            
total 0

-> # rpm -ivh kernel-3.10.0-957.el7.x86_64.rpm kernel-devel-3.10.0-957.el7.x86_64.rpm kernel-headers-3.10.0-957.el7.x86_64.rpm 
Preparing...                          ################################# [100%]
Updating / installing...
   1:kernel-headers-3.10.0-957.el7    ################################# [ 33%]
   2:kernel-devel-3.10.0-957.el7      ################################# [ 67%]
   3:kernel-3.10.0-957.el7            ################################# [100%]

# 查看安装情况
-> # rpm -qa |grep kernel                                                                                                      
kernel-devel-3.10.0-957.el7.x86_64
kernel-headers-3.10.0-957.el7.x86_64
kernel-3.10.0-957.el7.x86_64

# 查看内核文件
-> # ll /usr/src/kernels
total 4.0K
drwxr-xr-x 22 root root 4.0K Nov 22 09:33 3.10.0-957.el7.x86_64

# 查看内核模块情况
-> # ll /usr/lib/modules/3.10.0-957.el7.x86_64
total 3.2M
lrwxrwxrwx  1 root root   38 Nov 22 09:33 build -> /usr/src/kernels/3.10.0-957.el7.x86_64
drwxr-xr-x  2 root root    6 Nov  9  2018 extra
drwxr-xr-x 12 root root  128 Nov 22 09:33 kernel
-rw-r--r--  1 root root 821K Nov 22 09:33 modules.alias
-rw-r--r--  1 root root 784K Nov 22 09:33 modules.alias.bin
-rw-r--r--  1 root root 1.4K Nov  9  2018 modules.block
-rw-r--r--  1 root root 7.2K Nov  9  2018 modules.builtin
-rw-r--r--  1 root root 9.2K Nov 22 09:33 modules.builtin.bin
-rw-r--r--  1 root root 265K Nov 22 09:33 modules.dep
-rw-r--r--  1 root root 370K Nov 22 09:33 modules.dep.bin
-rw-r--r--  1 root root  361 Nov 22 09:33 modules.devname
-rw-r--r--  1 root root  132 Nov  9  2018 modules.drm
-rw-r--r--  1 root root   82 Nov  9  2018 modules.modesetting
-rw-r--r--  1 root root 1.8K Nov  9  2018 modules.networking
-rw-r--r--  1 root root  95K Nov  9  2018 modules.order
-rw-r--r--  1 root root  518 Nov 22 09:33 modules.softdep
-rw-r--r--  1 root root 382K Nov 22 09:33 modules.symbols
-rw-r--r--  1 root root 467K Nov 22 09:33 modules.symbols.bin
lrwxrwxrwx  1 root root    5 Nov 22 09:33 source -> build
drwxr-xr-x  2 root root    6 Nov  9  2018 updates
drwxr-xr-x  2 root root   95 Nov 22 09:33 vdso
drwxr-xr-x  2 root root    6 Nov 21 10:52 video
drwxr-xr-x  2 root root    6 Nov  9  2018 weak-updates

# 查看当前内核版本
uname -r 
3.10.0-1160.105.1.el7.x86_64

# 查看Linux 内核模块的位置，如果显着红色，说明内核文件不存，需要重装
ls -l /usr/lib/modules/ 

# 搜索一下非 160.105 版本
rpm -qa |grep kernel |grep -v 160.105

# 将上面的存在版本删除掉
rpm -qa |grep kernel |grep -v 160.105|xargs rpm -evh

yum install kernel kernel-devel kernel-header

-> #  sudo awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
0 : CentOS Linux (3.10.0-957.el7.x86_64) 7 (Core)
1 : CentOS Linux (0-rescue-xxxx) 7 (Core)

-> # grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-3.10.0-957.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-957.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-542ec60730c042d18738ae243747e2e0
Found initrd image: /boot/initramfs-0-rescue-542ec60730c042d18738ae243747e2e0.img
done

reboot

sh NVIDIA-Linux-x86_64-460.32.03.run --uninstall

# 驱动名称：NVIDIA-Linux-x86_64-460.32.03.run
sh NVIDIA-Linux-x86_64-460.32.03.run -no-x-check -no-nouveau-check -no-opengl-files

i) `rpm -i nvidia-driver-local-repo-rhel7-460.106.00-1.0-1.x86_64.rpm'
ii) `yum clean all`
iii) `yum install cuda-drivers`
iv) `reboot`

# nvidia-container-runtime
## 添加源
distribution=$(. /etc/os-release;echo $ID$VERSION_ID) 
curl -s -L https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.repo | \
sudo tee /etc/yum.repos.d/nvidia-container-runtime.repo

# CentOS
sudo yum -y install nvidia-container-runtime

# Ubuntu
sudo apt-get install nvidia-container-runtime

sudo mkdir -p /etc/systemd/system/docker.service.d
sudo tee /etc/systemd/system/docker.service.d/override.conf <<EOF
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd --host=fd:// --add-runtime=nvidia=/usr/bin/nvidia-container-runtime
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker

sudo tee /etc/docker/daemon.json <<EOF
{
    "default-runtime": "nvidia",
    "runtimes": {
        "nvidia": {
            "path": "/usr/bin/nvidia-container-runtime",
            "runtimeArgs": []
        }
    }
}
EOF
sudo pkill -SIGHUP dockerd

# CentOS7 为例，更多点击URL
curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | \
  sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo

sudo yum-config-manager --enable nvidia-container-toolkit-experimental

sudo yum install -y nvidia-container-toolkit

kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.12.3/nvidia-device-plugin.yml

# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nvidia-device-plugin-daemonset
  namespace: kube-system
spec:
  selector:
    matchLabels:
      name: nvidia-device-plugin-ds
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        name: nvidia-device-plugin-ds
    spec:
        nodeSelector:     # 新增行
        node-type: gpu  # 新增行
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
      # Mark this pod as a critical add-on; when enabled, the critical add-on
      # scheduler reserves resources for critical add-on pods so that they can
      # be rescheduled after a failure.
      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
      priorityClassName: "system-node-critical"
      containers:
      - image: nvcr.io/nvidia/k8s-device-plugin:v0.12.3
        name: nvidia-device-plugin-ctr
        env:
          - name: FAIL_ON_INIT_ERROR
            value: "false"
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop: ["ALL"]
        volumeMounts:
          - name: device-plugin
            mountPath: /var/lib/kubelet/device-plugins
      volumes:
        - name: device-plugin
          hostPath:
            path: /var/lib/kubelet/device-plugins