diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.0.3/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_cephcluster.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.0.3/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/patch_operator.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_operator.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.0.3/patch_operator.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_operator.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml new file mode 100644 index 0000000..a7fdd47 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml @@ -0,0 +1,11 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/storageclass.yaml + +patchesStrategicMerge: + - patch_operator.yaml + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml \ No newline at end of file diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_cephcluster.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.0/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml new file mode 100644 index 0000000..4e949ed --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml @@ -0,0 +1,13 @@ +# operator +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rook-ceph-operator +spec: + template: + spec: + containers: + - name: rook-ceph-operator + env: + - name: FLEXVOLUME_DIR_PATH + value: "/var/lib/kubelet/volume-plugins" \ No newline at end of file diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.0/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/patch_cephcluster.yaml similarity index 80% rename from clr-k8s-examples/7-rook/overlays/v1.1.1/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/patch_cephcluster.yaml index 194933a..29df390 100644 --- a/clr-k8s-examples/7-rook/overlays/v1.1.1/patch_cephcluster.yaml +++ b/clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/patch_cephcluster.yaml @@ -1,3 +1,4 @@ +--- apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -6,4 +7,4 @@ metadata: spec: storage: directories: - - path: /var/lib/rook \ No newline at end of file + - path: /var/lib/rook diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml new file mode 100644 index 0000000..07b29ff --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml + +patchesStrategicMerge: + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml new file mode 100644 index 0000000..be72622 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + mon: + allowMultiplePerNode: true + storage: + directories: + - path: /var/lib/rook +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + replicated: + requireSafeReplicaSize: false + size: 1 diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.1/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/patch_cephcluster.yaml similarity index 80% rename from clr-k8s-examples/7-rook/overlays/v1.1.7/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/patch_cephcluster.yaml index 194933a..29df390 100644 --- a/clr-k8s-examples/7-rook/overlays/v1.1.7/patch_cephcluster.yaml +++ b/clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/patch_cephcluster.yaml @@ -1,3 +1,4 @@ +--- apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -6,4 +7,4 @@ metadata: spec: storage: directories: - - path: /var/lib/rook \ No newline at end of file + - path: /var/lib/rook diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml new file mode 100644 index 0000000..07b29ff --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml + +patchesStrategicMerge: + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml new file mode 100644 index 0000000..be72622 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + mon: + allowMultiplePerNode: true + storage: + directories: + - path: /var/lib/rook +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + replicated: + requireSafeReplicaSize: false + size: 1 diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.7/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml new file mode 100644 index 0000000..29df390 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + storage: + directories: + - path: /var/lib/rook diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml new file mode 100644 index 0000000..07b29ff --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml + +patchesStrategicMerge: + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml new file mode 100644 index 0000000..be72622 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + mon: + allowMultiplePerNode: true + storage: + directories: + - path: /var/lib/rook +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + replicated: + requireSafeReplicaSize: false + size: 1 diff --git a/clr-k8s-examples/README.md b/clr-k8s-examples/README.md index e289e70..8753af5 100644 --- a/clr-k8s-examples/README.md +++ b/clr-k8s-examples/README.md @@ -162,3 +162,18 @@ Grafana is available at this URL http://localhost:3000 . Default credentials are ## Cleaning up the cluster (Hard reset to a clean state) Run `reset_stack.sh` on all the nodes + +## Additional Components + +### Rook +The default Rook configuration provided is intended for testing purposes only +and is not suitable for a production environment. By default Rook is configured +to provide local storage (/var/lib/rook) and will be provisioned differently +depending on whether or not you startup a single node Kubernetes cluster, or +a multiple node Kubernetes cluster. + +- When starting up a single node Kubernetes cluster, Rook will be configured +to start up a single replica, and will allow multiple monitors on the same node. +- When multiple Kubernetes worker nodes are detected, Rook will be configured +to startup a replica on each available node and will schedule monitor processes +on separate nodes providing greater reliability. diff --git a/clr-k8s-examples/create_stack.sh b/clr-k8s-examples/create_stack.sh index 4c7923a..9c2f470 100755 --- a/clr-k8s-examples/create_stack.sh +++ b/clr-k8s-examples/create_stack.sh @@ -33,6 +33,7 @@ CNI=${CLRK8S_CNI:-"canal"} if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-"crio"}"; fi NFD_VER="${CLRK8S_NFD_VER:-v0.4.0}" +mode="multinode" function print_usage_exit() { exit_code=${1:-0} @@ -108,6 +109,7 @@ function cluster_init() { #Ensure single node k8s works if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then kubectl taint nodes --all node-role.kubernetes.io/master- + mode="standalone" fi } @@ -208,9 +210,9 @@ function storage() { ROOK_DIR=7-rook # get and apply rook - get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}" - set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/rook" - kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}" + get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}" + set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook" + kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}" # wait for the rook OSDs to run which means rooks should be ready while [[ $(kubectl get po --all-namespaces | grep -e 'osd.*Running.*' -c) -lt 1 ]]; do echo "Waiting for Rook OSD"