Fix rook single node setup (#313)

* Fix rook single node setup

Modified the rook installation to support both multinode kubernetes
clusters and standalone kubernetes clusters. Multinode installations
will occur as before, with changes for standalone installations.
v0.8.3 I found it was not currently working and
is too old to continue to support, will submit a PR for removing it. Also made a couple minor spacing
changes to yaml as detected by yamllint

closes issue 306 https://github.com/clearlinux/cloud-native-setup/issues/306
closes issue 311 https://github.com/clearlinux/cloud-native-setup/issues/311

Signed-off-by: Craig Sterrett <craig.Sterrett@intel.com>

* Fix rook single node setup

Modified the rook installation to support both multinode kubernetes
clusters and standalone kubernetes clusters. Multinode installations
will occur as before, with changes for standalone installations.
v0.8.3 I found it was not currently working and
is too old to continue to support, will submit a PR for removing it. Also made a couple minor spacing
changes to yaml as detected by yamllint

closes issue 306 clearlinux#306
closes issue 311 clearlinux#311

Signed-off-by: Craig Sterrett <craig.Sterrett@intel.com>

clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/kustomization.yaml

@@ -0,0 +1,11 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/storageclass.yaml
+
+patchesStrategicMerge:
+  - patch_operator.yaml
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml

clr-k8s-examples/7-rook/overlays/v1.0.3/standalone/patch_operator.yaml

@@ -0,0 +1,13 @@
+# operator
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rook-ceph-operator
+spec:
+  template:
+    spec:
+      containers:
+        - name: rook-ceph-operator
+          env:
+          - name: FLEXVOLUME_DIR_PATH
+            value: "/var/lib/kubelet/volume-plugins"

clr-k8s-examples/7-rook/overlays/v1.1.0/multinode/patch_cephcluster.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: ceph.rook.io/v1
 kind: CephCluster
 metadata:
@@ -6,4 +7,4 @@ metadata:
 spec:
   storage:
     directories:
-     - path: /var/lib/rook
+      - path: /var/lib/rook

clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml

clr-k8s-examples/7-rook/overlays/v1.1.0/standalone/patch_cephcluster.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+  storage:
+    directories:
+      - path: /var/lib/rook
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1

clr-k8s-examples/7-rook/overlays/v1.1.1/multinode/patch_cephcluster.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: ceph.rook.io/v1
 kind: CephCluster
 metadata:
@@ -6,4 +7,4 @@ metadata:
 spec:
   storage:
     directories:
-     - path: /var/lib/rook
+      - path: /var/lib/rook

clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml

clr-k8s-examples/7-rook/overlays/v1.1.1/standalone/patch_cephcluster.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+  storage:
+    directories:
+      - path: /var/lib/rook
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1

clr-k8s-examples/7-rook/overlays/v1.1.7/multinode/patch_cephcluster.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  storage:
+    directories:
+      - path: /var/lib/rook

clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml

clr-k8s-examples/7-rook/overlays/v1.1.7/standalone/patch_cephcluster.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+  storage:
+    directories:
+      - path: /var/lib/rook
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1

clr-k8s-examples/README.md

@@ -162,3 +162,18 @@ Grafana is available at this URL http://localhost:3000 . Default credentials are
 ## Cleaning up the cluster (Hard reset to a clean state)
 
 Run `reset_stack.sh` on all the nodes
+
+## Additional Components
+
+### Rook
+The default Rook configuration provided is intended for testing purposes only
+and is not suitable for a production environment. By default Rook is configured
+to provide local storage (/var/lib/rook) and will be provisioned differently
+depending on whether or not you startup a single node Kubernetes cluster, or
+a multiple node Kubernetes cluster. 
+
+- When starting up a single node Kubernetes cluster, Rook will be configured
+to start up a single replica, and will allow multiple monitors on the same node.
+- When multiple Kubernetes worker nodes are detected, Rook will be configured
+to startup a replica on each available node and will schedule monitor processes
+on separate nodes providing greater reliability.

clr-k8s-examples/create_stack.sh

@@ -33,6 +33,7 @@ CNI=${CLRK8S_CNI:-"canal"}
 if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-"crio"}"; fi
 
 NFD_VER="${CLRK8S_NFD_VER:-v0.4.0}"
+mode="multinode"
 
 function print_usage_exit() {
 	exit_code=${1:-0}
@@ -108,6 +109,7 @@ function cluster_init() {
 	#Ensure single node k8s works
 	if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then
 		kubectl taint nodes --all node-role.kubernetes.io/master-
+		mode="standalone"
 	fi
 }
 
@@ -208,9 +210,9 @@ function storage() {
 	ROOK_DIR=7-rook
 
 	# get and apply rook
-	get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}"
-	set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/rook"
-	kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}"
+	get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}"
+	set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook"
+	kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}"
 	# wait for the rook OSDs to run which means rooks should be ready
 	while [[ $(kubectl get po --all-namespaces | grep -e 'osd.*Running.*' -c) -lt 1 ]]; do
 		echo "Waiting for Rook OSD"