From 514efd65924eb25e6e85422820aca62e8b3c8c0f Mon Sep 17 00:00:00 2001 From: CraigSterrett Date: Thu, 26 Mar 2020 10:07:18 -0700 Subject: [PATCH] Fix rook single node setup (#313) * Fix rook single node setup Modified the rook installation to support both multinode kubernetes clusters and standalone kubernetes clusters. Multinode installations will occur as before, with changes for standalone installations. v0.8.3 I found it was not currently working and is too old to continue to support, will submit a PR for removing it. Also made a couple minor spacing changes to yaml as detected by yamllint closes issue 306 https://github.com/clearlinux/cloud-native-setup/issues/306 closes issue 311 https://github.com/clearlinux/cloud-native-setup/issues/311 Signed-off-by: Craig Sterrett * Fix rook single node setup Modified the rook installation to support both multinode kubernetes clusters and standalone kubernetes clusters. Multinode installations will occur as before, with changes for standalone installations. v0.8.3 I found it was not currently working and is too old to continue to support, will submit a PR for removing it. Also made a couple minor spacing changes to yaml as detected by yamllint closes issue 306 clearlinux#306 closes issue 311 clearlinux#311 Signed-off-by: Craig Sterrett --- .../v1.0.3/{ => multinode}/kustomization.yaml | 0 .../{ => multinode}/patch_cephcluster.yaml | 0 .../{ => multinode}/patch_operator.yaml | 0 .../v1.0.3/standalone/kustomization.yaml | 11 ++++++++++ .../standalone}/patch_cephcluster.yaml | 0 .../v1.0.3/standalone/patch_operator.yaml | 13 +++++++++++ .../v1.1.0/{ => multinode}/kustomization.yaml | 0 .../multinode}/patch_cephcluster.yaml | 3 ++- .../v1.1.0/standalone/kustomization.yaml | 10 +++++++++ .../v1.1.0/standalone/patch_cephcluster.yaml | 22 +++++++++++++++++++ .../v1.1.1/{ => multinode}/kustomization.yaml | 0 .../multinode}/patch_cephcluster.yaml | 3 ++- .../v1.1.1/standalone/kustomization.yaml | 10 +++++++++ .../v1.1.1/standalone/patch_cephcluster.yaml | 22 +++++++++++++++++++ .../v1.1.7/{ => multinode}/kustomization.yaml | 0 .../v1.1.7/multinode/patch_cephcluster.yaml | 10 +++++++++ .../v1.1.7/standalone/kustomization.yaml | 10 +++++++++ .../v1.1.7/standalone/patch_cephcluster.yaml | 22 +++++++++++++++++++ clr-k8s-examples/README.md | 15 +++++++++++++ clr-k8s-examples/create_stack.sh | 8 ++++--- 20 files changed, 154 insertions(+), 5 deletions(-) rename clr-k8s-examples/7-rook/overlays/v1.0.3/{ => multinode}/kustomization.yaml (100%) rename clr-k8s-examples/7-rook/overlays/v1.0.3/{ => multinode}/patch_cephcluster.yaml (100%) rename clr-k8s-examples/7-rook/overlays/v1.0.3/{ => multinode}/patch_operator.yaml (100%) create mode 100644 clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml rename clr-k8s-examples/7-rook/overlays/{v1.1.0 => v1.0.3/standalone}/patch_cephcluster.yaml (100%) create mode 100644 clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml rename clr-k8s-examples/7-rook/overlays/v1.1.0/{ => multinode}/kustomization.yaml (100%) rename clr-k8s-examples/7-rook/overlays/{v1.1.1 => v1.1.0/multinode}/patch_cephcluster.yaml (80%) create mode 100644 clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml create mode 100644 clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml rename clr-k8s-examples/7-rook/overlays/v1.1.1/{ => multinode}/kustomization.yaml (100%) rename clr-k8s-examples/7-rook/overlays/{v1.1.7 => v1.1.1/multinode}/patch_cephcluster.yaml (80%) create mode 100644 clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml create mode 100644 clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml rename clr-k8s-examples/7-rook/overlays/v1.1.7/{ => multinode}/kustomization.yaml (100%) create mode 100644 clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml create mode 100644 clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml create mode 100644 clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.0.3/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_cephcluster.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.0.3/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/patch_operator.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_operator.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.0.3/patch_operator.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/multinode/patch_operator.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml new file mode 100644 index 0000000..a7fdd47 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml @@ -0,0 +1,11 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/storageclass.yaml + +patchesStrategicMerge: + - patch_operator.yaml + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml \ No newline at end of file diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_cephcluster.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.0/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml new file mode 100644 index 0000000..4e949ed --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml @@ -0,0 +1,13 @@ +# operator +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rook-ceph-operator +spec: + template: + spec: + containers: + - name: rook-ceph-operator + env: + - name: FLEXVOLUME_DIR_PATH + value: "/var/lib/kubelet/volume-plugins" \ No newline at end of file diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.0/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/patch_cephcluster.yaml similarity index 80% rename from clr-k8s-examples/7-rook/overlays/v1.1.1/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/patch_cephcluster.yaml index 194933a..29df390 100644 --- a/clr-k8s-examples/7-rook/overlays/v1.1.1/patch_cephcluster.yaml +++ b/clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/patch_cephcluster.yaml @@ -1,3 +1,4 @@ +--- apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -6,4 +7,4 @@ metadata: spec: storage: directories: - - path: /var/lib/rook \ No newline at end of file + - path: /var/lib/rook diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml new file mode 100644 index 0000000..07b29ff --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml + +patchesStrategicMerge: + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml new file mode 100644 index 0000000..be72622 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + mon: + allowMultiplePerNode: true + storage: + directories: + - path: /var/lib/rook +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + replicated: + requireSafeReplicaSize: false + size: 1 diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.1/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/patch_cephcluster.yaml similarity index 80% rename from clr-k8s-examples/7-rook/overlays/v1.1.7/patch_cephcluster.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/patch_cephcluster.yaml index 194933a..29df390 100644 --- a/clr-k8s-examples/7-rook/overlays/v1.1.7/patch_cephcluster.yaml +++ b/clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/patch_cephcluster.yaml @@ -1,3 +1,4 @@ +--- apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -6,4 +7,4 @@ metadata: spec: storage: directories: - - path: /var/lib/rook \ No newline at end of file + - path: /var/lib/rook diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml new file mode 100644 index 0000000..07b29ff --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml + +patchesStrategicMerge: + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml new file mode 100644 index 0000000..be72622 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + mon: + allowMultiplePerNode: true + storage: + directories: + - path: /var/lib/rook +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + replicated: + requireSafeReplicaSize: false + size: 1 diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/kustomization.yaml similarity index 100% rename from clr-k8s-examples/7-rook/overlays/v1.1.7/kustomization.yaml rename to clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/kustomization.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml new file mode 100644 index 0000000..29df390 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + storage: + directories: + - path: /var/lib/rook diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml new file mode 100644 index 0000000..07b29ff --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - rook/cluster/examples/kubernetes/ceph/common.yaml + - rook/cluster/examples/kubernetes/ceph/operator.yaml + - rook/cluster/examples/kubernetes/ceph/cluster.yaml + - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml + +patchesStrategicMerge: + # patches rook to use 'directories' instead of partitions. + # comment out to use partitions + - patch_cephcluster.yaml diff --git a/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml new file mode 100644 index 0000000..be72622 --- /dev/null +++ b/clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + mon: + allowMultiplePerNode: true + storage: + directories: + - path: /var/lib/rook +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + replicated: + requireSafeReplicaSize: false + size: 1 diff --git a/clr-k8s-examples/README.md b/clr-k8s-examples/README.md index e289e70..8753af5 100644 --- a/clr-k8s-examples/README.md +++ b/clr-k8s-examples/README.md @@ -162,3 +162,18 @@ Grafana is available at this URL http://localhost:3000 . Default credentials are ## Cleaning up the cluster (Hard reset to a clean state) Run `reset_stack.sh` on all the nodes + +## Additional Components + +### Rook +The default Rook configuration provided is intended for testing purposes only +and is not suitable for a production environment. By default Rook is configured +to provide local storage (/var/lib/rook) and will be provisioned differently +depending on whether or not you startup a single node Kubernetes cluster, or +a multiple node Kubernetes cluster. + +- When starting up a single node Kubernetes cluster, Rook will be configured +to start up a single replica, and will allow multiple monitors on the same node. +- When multiple Kubernetes worker nodes are detected, Rook will be configured +to startup a replica on each available node and will schedule monitor processes +on separate nodes providing greater reliability. diff --git a/clr-k8s-examples/create_stack.sh b/clr-k8s-examples/create_stack.sh index 4c7923a..9c2f470 100755 --- a/clr-k8s-examples/create_stack.sh +++ b/clr-k8s-examples/create_stack.sh @@ -33,6 +33,7 @@ CNI=${CLRK8S_CNI:-"canal"} if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-"crio"}"; fi NFD_VER="${CLRK8S_NFD_VER:-v0.4.0}" +mode="multinode" function print_usage_exit() { exit_code=${1:-0} @@ -108,6 +109,7 @@ function cluster_init() { #Ensure single node k8s works if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then kubectl taint nodes --all node-role.kubernetes.io/master- + mode="standalone" fi } @@ -208,9 +210,9 @@ function storage() { ROOK_DIR=7-rook # get and apply rook - get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}" - set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/rook" - kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}" + get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}" + set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook" + kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}" # wait for the rook OSDs to run which means rooks should be ready while [[ $(kubectl get po --all-namespaces | grep -e 'osd.*Running.*' -c) -lt 1 ]]; do echo "Waiting for Rook OSD"