内存     硬盘     cpu

master 10G 100G 4

node 8G 100G 4

k8s

使用kubeeasy

搭建k8s

使用 kubeeasy 工具 完成 Kubernetes 1.22.1 集群的搭建

1. 设置主机名master、node
2. 关闭防火墙selinux
3. 配置hosts文件
4. 配置免密
5. master 挂载镜像chinaskills_cloud_paas_v2.0.2.iso

mount /dev/sr0 /mnt -o loop
cp -rfv /mnt/* /opt/         # 替换base-rpms.tar.gz到/mnt/dependencies/里面
umount

3. master 将kubeeasy复制到/usr/bin，安装kubeeasy

cp -rf /opt/kubeeasy /usr/bin/
kubeeasy install dependencies --host 192.168.100.10,192.168.100.11 --user root --password 000000 --offline-file /opt/dependencies/base-rpms.tar.gz
kubeeasy install kubernetes --master 192.168.100.10 --worker 192.168.100.11 --user root --password 000000 --version 1.22.1 --offline-file /opt/kubernetes.tar.gz

# 日志查看
tail -f /var/log/messages & kubeinstall.log

创建pod

然后使用 nginx 镜像在 default 命名空间下创建一个名为 exam 的 Pod，并为该 Pod 设置环境变量 exam，其值为 2022

kubectl run exam --image nginx:latest --env="exam=2022" --image-pull-policy='IfNotPresent'

部署 Istio 服务网格

在 Kubernetes 集群上完成 Istio 服务网格环境的安装，然后新建命名空间 exam，为该命 名空间开启自动注入 Sidecar

#网络安装
kubeeasy add --istio istion
#创建namespace
kubectl create namespace exam
#自动注入sidecar
kubectl label ns exam istio-injection=enabled

部署 KubeVirt 虚拟化

在 Kubernetes 集群上完成 KubeVirt 虚拟化环境的安装

kubeeasy add --virt kubevirt

部署 Harbor 仓库

安装部署

在 master 节点上完成 Harbor 镜像仓库及 Helm 包管理工具的部署。

kubeeasy add --registry harbor

使用deployment

然后使用 nginx 镜像 自定义一个 Chart，Deployment 名称为 nginx，副本数为 1，然后将该 Chart 部署到 default 命名空间下，Release 名称为 web

# 需要在工作目录有nginx.tar.gz
helm create nginx    # 使用helm工具创建nginx项目
cd nginx
rm -rf templates/*
#创建deployment模板
kubectl create deployment nginx -o yaml --image=nginx:latest --replicas=1 --dry-run=client --port=80 > /root/nginx/templates/deployment.yaml

# 在image name后面添加查找镜像
vi /templates/deployment.yaml
imagePullPolicy: IfNotPresent

vi templates/NOTES.txt
name:nginx

cd ..
helm install web nginx

卸载 heml uninstall web

备份 ETCD 数据

Kubernetes 使用 ETCD 来存储集群的实时运行数据，为防止服务器宕机导致 Kubernetes 集群数据丢失，请将 Kubernetes 集群数据备份到/root/etcd.db 中

[root@k8s-master-node1 ~]# docker ps -a | grep etcd
9a8e4e589662   004811815584                         "etcd --advertise-cl…"   47 minutes ago   Up 47 minutes                                                       k8s_etcd_etcd-k8s-master-node1_kube-system_359e897a01149111969d3f5d475f8e96_0
39ae1627794c   k8s.gcr.io/pause:3.5                 "/pause"                 47 minutes ago   Up 47 minutes                                                       k8s_POD_etcd-k8s-master-node1_kube-system_359e897a01149111969d3f5d475f8e96_0
[root@k8s-master-node1 ~]# docker cp 9a8e4e589662:/usr/local/bin/etcdctl /usr/local/bin/etcdctl

CICD

安装 GitLab 环境

在 Kubernetes 集群中新建命名空间 gitlab-ci，将 GitLab 部署到该命名空间下，Deployment 和 Service 名称均为 gitlab，以 NodePort 方式将 80 端口对外暴露为 30880，设置 GitLab 服务 root 用户的密码为 admin@123，将项目包 demo-2048.tar.gz 导入到 GitLab 中并命名为 demo-2048。

apiVersion: apps/v1
kind: Deployment
metadata:
  name: gitlab
  namespace: gitlab-ci
  labels: 
    name: gitlab
spec:
  selector:
    matchLabels:
      name: gitlab
  template:
    metadata:
      name: gitlab
      labels:
        name: gitlab
    spec:
      containers:
      - image: gitlab/gitlab-ce:latest
        name: gitlab
        imagePullPolicy: IfNotPresent
        env:
        - name: GITLAB_ROOT_PASSWORD
          value: admin@123
        - name: GITLAB_HOST
          value: 192.168.150.100
        - name: GITLAB_PORT
          value: "80"
        ports:
        - name: http
          containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: gitlab
  namespace: gitlab-ci
  labels:
    name: gitlab
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      targetPort: http
      nodePort: 30880
  selector:
    name: gitlab

安装jenkins

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: jenkins
  labels:
    name: jenkins
spec:
  selector:
    matchLabels:
       name: jenkins
  template:
    metadata:
      name: jenkins
      labels:
        name: jenkins
    spec:
      containers:
      - name: jenkins
        image: jenkins/jenkins:2.262-centos
        ports:
        - name: http
          containerPort: 8080
        - name: other
          containerPort: 50000
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: jenkins
  labels:
    name: jenkins
spec:
  type: NodePort
  ports:
  - name: http
    nodePort: 8080
    targetPort: http
    port: 8080
  selector:
    name: jenkins

容器云服务运维

时间同步

apiVersion: v1
kind: Pod
metadata:
  name: time
spec:
  containers:
  - name: time
    image: 192.168.59.22/library/nginx:latest
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - name: timefile
      mountPath: /etc/localtime
  volumes:
  - name: timefile
    hostPath:
      path: /etc/localtime

secret

[root@master ~]# echo -n 'admin' | base64          
YWRtaW4=
[root@master ~]# echo -n '1f2d1e2e67df' | base64
MWYyZDFlMmU2N2Rm

apiVersion: v1
kind: Secret
metadata:
  name: secret
type: Opaque
data:
  username: YWRtaW4=
  password: MWYyZDFlMmU2N2Rm

资源限制

在 default 命名空间下使用 nginx:latest 镜像创建一个 QoS 类为 Guaranteed 的 Pod，名称 为 qos-demo。

kubectl create -f qos-demo.yaml

apiVersion: v1
kind: Pod
metadata:
  labels:
    run: pod
  name: qos-demo
spec:
  containers:
  - image: nginx:latest
    imagePullPolicy: IfNotPresent
    name: pod
    ports:
    - containerPort: 80
    resources:
      limits:
        cpu: "100m"
        memory: "100Mi"
      requests:
        cpu: "100m"
        memory: "100Mi"

安全上下文

使用 busybox 镜像启动一个名为 context-demo 的 Pod，为该 Pod 配置安全上下文，要求 容器内以用户 1000 和用户组 3000 来运行所有进程，并在启动时执行“sleep 1h”命令。

kubectl apply -f context-demo.yaml

apiVersion: v1
kind: Pod
metadata:
  labels:
    run: context-demo
  name: context-demo
spec:
  securityContext:
    runAsUser: 1000
    runAsGroup: 3000
  containers:
  - image: busybox:latest
    name: context-demo
    imagePullPolicy: IfNotPresent
    command: [ "sh", "-c", "sleep 1h" ]
    securityContext:
     allowPrivilegeEscalation: false
  nodeSelector:
    kubernetes.io/hostname: k8s-master-node1

创建自定义资源类型

在 Kubernetes 集群中自定义一种资源类型 Student，API 为 stable.example.com/v1，单数 形式为 student，复数形式为 students，简写为 stu，作用域为命名空间级，然后在 default 命 名空间下创建一个名为 exam 的 Student 对象。

kubectl apply -f crd.yaml

apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: students.stable.example.com
spec:
  group: stable.example.com
  versions:
    - name: v1
      served: true
      storage: true
      schema:
          openAPIV3Schema:
            type: object
            properties:
              spec:
                type: object
              tags:
                type: string
  scope: Namespaced
  names:
    plural: students
    kind: Student
    shortNames:
    - stu

# kubectl get crd
NAME                          CREATED AT
students.stable.example.com   2023-02-27T01:47:08Z

kubectl apply -f stu.yaml

apiVersion: stable.example.com/v1
kind: Student
metadata:
  name: exam
spec:
  tags: test

添加主机别名到 Pod

使用 nginx 镜像在 default 命名空间下创建一个名为 nginx 的 Pod，并在 Pod 的/etc/hosts 中添加 IP 地址 127.0.0.1 与 chinaskills 的解析。

kubectl apply -f pod-host.yaml

apiVersion: v1
kind: Pod
metadata:
  labels:
    run: nginx
  name: nginx
spec:
  containers:
  - image: nginx:latest
    imagePullPolicy: IfNotPresent
    name: nginx
    ports:
    - containerPort: 80
  hostAliases:
  - ip: 127.0.0.1
    hostnames:
    - chinaskills

kubectl exec -it nginx -- cat /etc/hosts

创建 HPA 规则

默认情况下 HPA 是无法调整伸缩灵敏度的，但不同的业务场景对伸缩灵敏度的要求不 一样。要求在 default 命名空间下使用 nginx 镜像创建一个名为 web 的 deployment，自定义 HPA 的伸缩灵敏度，为该 deployment 创建一个名为 web 的 HPA，扩容时立即新增当前 9 倍 数量的副本数，时间窗口为 5s，伸缩范围为 1–1000。例如一开始只有 1 个 Pod，当 CPU 使 用率超过 80%时，Pod 数量变化趋势为：1 → 10 → 100 → 1000

kubectl apply -f web-hpa.yaml
kubectl apply -f hpa.yaml

apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
  name: web
spec:
  minReplicas: 1
  maxReplicas: 1000
  metrics:
   - type: Resource
     resource:
         name: cpu
         target :
             type: Utilization
             averageUtilization: 80
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: web
  behavior:
   scaleUp:
     stabilizationWindowSeconds: 5
     policies:
     - type: Percent
       value: 900
       periodSeconds: 5

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: web
  name: web
spec:
  replicas: 1
  selector:
    matchLabels:
      app: web
  template:
    metadata:
      labels:
        app: web
    spec:
      containers:
      - image: nginx:latest
        imagePullPolicy: IfNotPresent
        name: nginx
        ports:
        - containerPort: 80
        resources:
          limits:
            cpu: "10m"
            memory: "20Mi"
          requests:
            cpu: "10m"
            memory: "20Mi"

[root@k8s-master-node1 4]# kubectl describe hpa
Name:                                                  web
Namespace:                                             default
Labels:                                                <none>
Annotations:                                           <none>
CreationTimestamp:                                     Mon, 27 Feb 2023 10:07:25 +0800
Reference:                                             Deployment/web
Metrics:                                               ( current / target )
  resource cpu on pods  (as a percentage of request):  <unknown> / 80%
Min replicas:                                          1
Max replicas:                                          1000
Behavior:
  Scale Up:
    Stabilization Window: 5 seconds
    Select Policy: Max
    Policies:
      - Type: Percent  Value: 900  Period: 5 seconds
  Scale Down:
    Select Policy: Max
    Policies:
      - Type: Percent  Value: 100  Period: 15 seconds
Deployment pods:       0 current / 0 desired
Conditions:
  Type         Status  Reason          Message
  ----         ------  ------          -------
  AbleToScale  False   FailedGetScale  the HPA controller was unable to get the target's current scale: no matches for kind "Deployment" in group "app"
Events:
  Type     Reason          Age                  From                       Message
  ----     ------          ----                 ----                       -------
  Warning  FailedGetScale  10s (x9 over 2m10s)  horizontal-pod-autoscaler  no matches for kind "Deployment" in group "app"

标签Pod

在 default 命名空间下使用 nginx 镜像运行一个 Pod，名称为 nginx，要求使用 requiredDuringSchedulingIgnoredDuringExecution 策略将 Pod 调度到具有“disktype=ssd”标 签的节点。

kubectl label node k8s-master-node1 disktype=ssd

apiVersion: v1
kind: Pod
metadata:
  labels:
    run: nginx
  name: nginx
spec:
  containers:
  - image: nginx:latest
    imagePullPolicy: IfNotPresent
    name: nginx
    ports:
    - containerPort: 80
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
            - key: disktype
              operator: In
              values:
              - ssd

[root@k8s-master-node1 4]# kubectl describe nodes k8s-master-node1  | grep ssd
                    disktype=ssd

创建 Pod 网络策略

创建一个网络策略 network-exam，要求只有 internal 命名空间下的 Pod 可以通过 TCP 协议的 8080 端口访问到 mysql 命名空间下的 Pod。

kubectl create ns mysql
kubectl create ns internal
kubectl apply -f network-exam.yaml

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: network-exam
  namespace: internal
spec:
  podSelector: {}
  policyTypes:
    - Ingress
  ingress:
    - from:
      - namespaceSelector:
          matchLabels:
            kubernetes.io/metadata.name: internal
      ports:
      - port: 8080
        protocol: TCP

驱逐机制管理

设置 kubelet数据存储在/apps/data/kubelet目录下，并设置当 kubelet的存储空间不足 5%， 或者当容器运行时文件系统可用存储空间不足 5%时开始驱逐 Pod

vi /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS=--root-dir=/apps/data/kubelet --eviction-hard
=nodefs.available<5% --eviction-hard=imagefs.available<5%
[root@k8s-master-node1 ~]# systemctl daemon-reload && systemctl restart kubelet

创建 Ingress Gateway

使用提供的软件包 ServiceMesh.tar.gz 将 Bookinfo 应用部署到 default 命名空间下，使用 Istio Gateway 可 以实 现应 用程 序从 外部 访问， 请为 Bookinfo 应用创 建一 个名 为 bookinfo-gateway 的网关，指定所有 HTTP 流量通过 80 端口流入网格，然后将网关绑定到虚 拟服务 bookinfo 上

[root@k8s-master-node1 ~]# cd ServiceMesh/images/
[root@k8s-master-node1 images]# scp image.tar 192.168.59.23:/root/
[root@k8s-master-node1 images]# docker load -i image.tar 
[root@k8s-worker-node1 ~]# docker load -i image.tar 
[root@k8s-master-node1 ~]# kubectl label ns default istio-injection=enable
[root@k8s-master-node1 ~]# cd ServiceMesh/bookinfo/
[root@k8s-master-node1 bookinfo]# kubectl apply -f bookinfo.yaml
[root@k8s-master-node1 bookinfo]# kubectl apply -f ratings-v2.yaml 
[root@k8s-master-node1 bookinfo]# kubectl apply -f reviews-v2.yaml 
[root@k8s-master-node1 bookinfo]# kubectl apply -f reviews-v3.yaml 
[root@k8s-master-node1 4]# kubectl apply -f bookinfo.yaml 

apiVersion: networking.istio.io/v1beta1
kind: Gateway
metadata:
     name: bookinfo-gateway
spec:
    selector:
        istio: ingressgateway
    servers:
    - port:
           number: 80
           name: http
           protocol: HTTP
      hosts:
      - "*"
---
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
    name: bookinfo
spec:
    hosts:
    - "*"
    gateways:
    - bookinfo-gateway
    http:
    - match:
       - uri:
             exact: /productpage
       - uri:
             prefix: /static
       - uri:
             exact: /login
       - uri:
             exact: /logout
       - uri:
             prefix: /api/v1/products
      route:
       - destination:
               host: productpage
               port:
                   number: 9080

创建基于用户身份的路由

创建一个名为 reviews 路由，要求来自名为 Jason 的用户的所有流量将被路由到服务 reviews:v2。

[root@k8s-master-node1 4]# kubectl apply -f review.yaml      

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
     name: reviews
spec:
    host: reviews
    subsets:
     - name: v1
       labels:
          version: v1
     - name: v2
       labels:
          version: v2
     - name: v3
       labels:
          version: v3
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: reviews
spec:
  hosts:
    - reviews
  http:
  - match:
    - headers:
        end-user:
          exact: jason
    route:
    - destination:
        host: reviews
        subset: v2
  - route:

基于虚拟机的运维

创建 VM

使用镜像 fedora-virt:v1.0 在 default 命名空间下创建一台 vm，名称为 vm-fedora，内存为 1G。

[root@k8s-master-node1 ~]# docker load -i fedora-virt_v1.0.tar 
[root@k8s-master-node1 5]# kubectl apply -f fedora-vm.yaml

apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
     labels:
        kubevirt.io/vm: vm-fedora
     name: vm-fedora
spec:
   runStrategy: Always
   template:
      metadata:
         labels:
            kubevirt.io/vm: vm-fedora
      spec:
          domain:
           resources:
               requests:
                   memory: 1Gi
           devices:
               disks:
               - name: containerdisk
                 disk:
                    bus: virtio
          volumes:
           - name: containerdisk
             containerDisk:
                 image: fedora-virt:v1.0

创建 emptyDisk 卷

使用镜像 fedora-virt:v1.0 在 default 命名空间下创建一台 vmi，名称为 vmi-fedora，并使 用 emptyDisk 卷为 vmi 挂载一块 2G 的磁盘

[root@k8s-master-node1 5]# kubectl apply -f vim-fedora.yaml

apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
    labels:
       special: vmi-fedora
    name: vmi-fedora
spec:
    domain:
        devices:
            disks:
             - disk:
                  bus: virtio
               name: containerdisk
             - name: emptydisk
               disk :
                 bus: virtio
        resources:
            requests:
               memory: 1Gi
    terminationGracePeriodSeconds: 0
    volumes:
    - containerDisk:
           image: fedora-virt:v1.0
      name: containerdisk
    - name: emptydisk
      emptyDisk:
           capacity: 2Gi

创建 VMI

将提供的镜像 exam.qcow2 转换为 docker 镜像 exam:v1.0，然后使用镜像 exam:v1.0 镜像 在 default 命名空间下创建一台 vmi，名称为 exam，将虚拟机的 80 端口以 NodePort 的方式 对外暴露为 30082，并使用数据源在启动时将 VM 的主机名初始化为 exam

docker build -t exam:v1.0 .

FROM scratch
ADD exam.qcow2 /disk/

kubectl apply -f exam.yaml

apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
    name: exam
    labels:
       app: exam
spec:
    domain:
        devices:
            disks:
             - name: containerdisk
               disk:
                  bus: virtio
             - name: cloudinitdisk
               disk:
                  bus: virtio
        resources:
            requests:
                memory: 512Mi
    volumes:
    - name: containerdisk
      containerDisk:
          image: exam:v1.0
          imagePullPolicy: IfNotPresent
    - name: cloudinitdisk
      cloudInitNoCloud:
                userData: |-
                    hostname: exam
---
apiVersion: v1
kind: Service
metadata:
     labels:
        app: exam
     name: exam
spec:
    ports:
    - name: 80-80
      nodePort: 30082
      port: 80
      protocol: TCP
      targetPort: 80
    selector:
        app: exam
    type: NodePort