k8s Cluster Integration with Ceph Cluster Deployment Guide

Task Objectives#

Complete the integration of the k8s cluster with the ceph cluster.

Task Platform#

Physical devices--
Operating System: openEuler 22.03 LTS SP2

Deployment Guide#

Use dynamic mounting of ceph RBD storage.

Task 1: Install Ceph Client (ceph-common)#

Install ceph-common on each node of the k8s cluster.

yum install ceph-common -y

Task 2: Synchronize Ceph Cluster Configuration Files#

Synchronize the ceph cluster's /etc/ceph/{ceph.conf,ceph.client.admin.keyring} files to all nodes in k8s.

# Configure SSH passwordless access
ssh-keygen -t rsa
ssh-copy-id 10.10.3.121
ssh-copy-id 10.10.3.122
ssh-copy-id 10.10.3.123
ssh-copy-id 10.10.3.124

# Copy files
scp -r /etc/ceph/{ceph.conf,ceph.client.admin.keyring} 10.10.3.121:/etc/ceph
scp -r /etc/ceph/{ceph.conf,ceph.client.admin.keyring} 10.10.3.122:/etc/ceph
scp -r /etc/ceph/{ceph.conf,ceph.client.admin.keyring} 10.10.3.123:/etc/ceph
scp -r /etc/ceph/{ceph.conf,ceph.client.admin.keyring} 10.10.3.124:/etc/ceph

Task 3: Deploy Ceph-CSI (Using RBD)#

Download the ceph-csi components.

# Download files
wget https://github.com/ceph/ceph-csi/archive/refs/tags/v3.9.0.tar.gz
# Unzip
mv v3.9.0.tar.gz ceph-csi-v3.9.0.tar.gz
tar -xzf ceph-csi-v3.9.0.tar.gz
# Enter directory
cd ceph-csi-3.9.0/deploy/rbd/kubernetes
mkdir /data/cephfs/csi
# Copy into csi, a total of six files
cp * /data/cephfs

Pull the required images for the CSI components.

# Check required images
grep image csi-rbdplugin-provisioner.yaml
grep image csi-rbdplugin.yaml

Pull the required images on all k8s nodes.

./pull-images.sh registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
./pull-images.sh registry.k8s.io/sig-storage/csi-resizer:v1.8.0
./pull-images.sh registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2
docker pull quay.io/cephcsi/cephcsi:v3.9.0
./pull-images.sh registry.k8s.io/sig-storage/csi-attacher:v4.3.0
./pull-images.sh registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0

Create the namespace cephfs.

echo '
apiVersion: v1
kind: Namespace
metadata:
  labels:
    kubernetes.io/metadata.name: cephfs
  name: cephfs
  ' > ceph-namespace.yaml
  
 # Execute
 kubectl apply -f ceph-namespace.yaml

Create the secret file to connect to the ceph cluster, csi-rbd-secret.yaml.

echo '
apiVersion: v1
kind: Secret
metadata:
  name: csi-rbd-secret
  namespace: cephfs
stringData:
  adminID: admin  # Admin name
  adminKey: AQCNNDZlPKI5KxAA/3gXnOmhHgyLU2qPzTaZ3A==   # Ceph cluster key
  userID: admin   # User name 
  userKey: AQCNNDZlPKI5KxAA/3gXnOmhHgyLU2qPzTaZ3A==  # Ceph cluster key
  ' > csi-rbd-secret.yaml
  
   # Execute
   kubectl apply -f csi-rbd-secret.yaml

Create ceph-config-map.yaml.

echo '
apiVersion: v1
kind: ConfigMap
data:
  ceph.conf: |
     [global]
     fsid = 79fd2206-39ca-4ec4-9cd2-96e065c6361e     # Generated FSID
     mon initial members =future-ceph-node0,future-ceph-node1,future-ceph-node2,future-ceph-node3            # Hostnames
     mon host = 10.10.3.117,10.10.3.118,10.10.3.119,10.10.3.120                       # Corresponding IPs
     public network = 10.10.3.0/24
     auth cluster required = cephx
     auth service required = cephx
     auth client required = cephx
     osd journal size = 1024
     osd pool default size = 3
     osd pool default min size = 2
     osd pool default pg num = 333
     osd pool default pgp num = 333
     osd crush chooseleaf type = 1
     [mon]
     mon allow pool delete = true

     [mds.future-ceph-node0]
     host = future-ceph-node0
  keyring: |
metadata:
  name: ceph-config
  namespace: cephfs
' > ceph-config-map.yaml

 # Execute
 kubectl apply -f ceph-config-map.yaml

Modify csi-config-map.yaml to configure the connection information for the ceph cluster.

echo '
apiVersion: v1
kind: ConfigMap
metadata:
  name: ceph-csi-config
  namespace: cephfs
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
  config.json: |-
    [{"clusterID":"79fd2206-39ca-4ec4-9cd2-96e065c6361e","monitors":["10.10.3.117:6789","10.10.3.118:6789","10.10.3.119:6789","10.10.3.120:6789"]}]
' > csi-config-map.yaml

Modify the CSI component configuration files.
1. Change the namespace from default to cephfs in all yaml files in the /data/cephfs/csi directory.
```
  cd /data/cephfs/csi
  sed -i "s/namespace: default/namespace: cephfs/g" $(grep -rl "namespace: default" ./)
  sed -i '/^kind: ServiceAccount/a\  namespace: cephfs' $(grep -rl "^kind: ServiceAccount" ./)
```
Comment out the KMS configuration part in csi-rbdplugin-provisioner.yaml and csi-rbdplugin.yaml.

# - name: KMS_CONFIGMAP_NAME

# value: encryptionConfig

#- name: ceph-csi-encryption-kms-config

# configMap:

# name: ceph-csi-encryption-kms-config

 # Execute to install the CSI components
 kubectl apply -f csi-config-map.yaml
 kubectl apply -f csi-nodeplugin-rbac.yaml
 kubectl apply -f csidriver.yaml
 kubectl apply -f csi-provisioner-rbac.yaml
 kubectl apply -f csi-rbdplugin-provisioner.yaml
 kubectl apply -f csi-rbdplugin.yaml

Task 4: Create StorageClass#

echo '
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    k8s.kuboard.cn/storageType: cephfs_provisioner
  name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:
 # fsName: cephfs # Ceph cluster cephfs filesystem name
  clusterID: 79fd2206-39ca-4ec4-9cd2-96e065c6361e # FSID of the ceph cluster 
  pool: rbd-k8s # Storage pool created in ceph
  imageFeatures: layering
  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
  csi.storage.k8s.io/provisioner-secret-namespace: cephfs
  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: cephfs
  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
  csi.storage.k8s.io/node-stage-secret-namespace: cephfs
  csi.storage.k8s.io/fstype: xfs
reclaimPolicy: Delete
volumeBindingMode: Immediate
#allowVolumeExpansion: true
#mountOptions:
 # - discard
 ' > storageclass.yaml
 
 # Execute
  kubectl apply -f storageclass.yaml

Task 5: Create PVC#

echo '
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rbd-pvc # PVC name
  namespace: cephfs
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi  # Set capacity
  storageClassName: csi-rbd-sc
  ' > pvc.yaml
  
# Execute
 kubectl apply -f pvc.yaml
# Check if PVC was created successfully
kubectl get pvc -n cephfs
# Check if PV was created successfully
kubectl get pv -n cephfs

# Check if an image was created in the cephfs_data storage pool in the ceph cluster
rbd ls -p cephfs_data

Task 6: Create Pod for Testing and Verification#

echo '
apiVersion: v1
kind: Pod
metadata:
  name: csi-rbd-demo-pod
  namespace: cephfs
spec:
  containers:
    - name: web-server
      image: nginx:latest
      volumeMounts:
        - name: mypvc
          mountPath: /var/lib/www/html
  volumes:
    - name: mypvc
      persistentVolumeClaim:
        claimName: rbd-pvc  # Corresponding PVC name
        readOnly: false
' > pod.yaml

# Execute
 kubectl apply -f pod.yaml
 # Enter the container to check the mount information
 kubectl exec -it csi-rbd-demo-pod bash
 lsblk -l|grep rbd