Finish adding ansible files for Scylla management (management stuff, not the actual cluster nodes)

ansible.cfg

@@ -0,0 +1,5 @@
+[ssh_connection]
+ssh_args = -o ControlMaster=auto -o ControlPersist=60s
+
+[defaults]
+timeout = 30

inventory.ini

@@ -1,2 +1,2 @@
 [dev]
-dev-node ansible_host=p4.vps.nyeki.dev ansible_user=root
+p4 ansible_host=p4.vps.nyeki.dev ansible_user=root

k8s/scylla/04-scylla-node-config.yml

@@ -0,0 +1,26 @@
+apiVersion: scylla.scylladb.com/v1alpha1
+kind: NodeConfig
+metadata:
+  name: scylladb-nodepool-1
+spec:
+  localDiskSetup:
+    loopDevices:
+    - name: persistent-volumes
+      imagePath: /var/lib/persistent-volumes.img
+      size: 50Gi
+    filesystems:
+    - device: /dev/loops/persistent-volumes
+      type: xfs
+    mounts:
+    - device: /dev/loops/persistent-volumes
+      mountPoint: /var/lib/persistent-volumes
+      unsupportedOptions:
+      - prjquota
+  placement:
+    nodeSelector:
+      scylla.scylladb.com/node-type: scylla
+    tolerations:
+    - effect: NoSchedule
+      key: scylla-operator.scylladb.com/dedicated
+      operator: Equal
+      value: scyllaclusters

k8s/scylla/05-1-scylla-csi-cluster-role.yml

@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: scylladb:csi-external-provisioner
+aggregationRule:
+  clusterRoleSelectors:
+  - matchLabels:
+      rbac.operator.scylladb.com/aggregate-to-csi-external-provisioner: "true"

k8s/scylla/05-2-scylla-csi-cluster-role-def.yml

@@ -0,0 +1,102 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scylladb:aggregate-to-csi-external-provisioner
+  labels:
+    rbac.operator.scylladb.com/aggregate-to-csi-external-provisioner: "true"
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "persistentvolumes"
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+  - "create"
+  - "delete"
+- apiGroups:
+  - ""
+  resources:
+  - "persistentvolumeclaims"
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+  - "update"
+- apiGroups:
+  - "storage.k8s.io"
+  resources:
+  - "storageclasses"
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+- apiGroups:
+  - ""
+  resources:
+  - "events"
+  verbs:
+  - "list"
+  - "watch"
+  - "create"
+  - "update"
+  - "patch"
+- apiGroups:
+  - "snapshot.storage.k8s.io"
+  resources:
+  - "volumesnapshots"
+  verbs:
+  - "get"
+  - "list"
+- apiGroups:
+  - "snapshot.storage.k8s.io"
+  resources:
+  - "volumesnapshotcontents"
+  verbs:
+  - "get"
+  - "list"
+- apiGroups:
+  - "storage.k8s.io"
+  resources:
+  - "csinodes"
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+- apiGroups:
+  - ""
+  resources:
+  - "nodes"
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+- apiGroups:
+  - "storage.k8s.io"
+  resources:
+  - "csistoragecapacities"
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+  - "create"
+  - "update"
+  - "patch"
+  - "delete"
+# The GET permissions below are needed for walking up the ownership chain
+# for CSIStorageCapacity. They are sufficient for deployment via
+# StatefulSet (only needs to get Pod) and Deployment (needs to get
+# Pod and then ReplicaSet to find the Deployment).
+- apiGroups:
+  - ""
+  resources:
+  - "pods"
+  verbs:
+  - "get"
+- apiGroups:
+  - "apps"
+  resources:
+  - "replicasets"
+  verbs:
+  - "get"

k8s/scylla/05-3-scylla-csi-cluster-role-def-openshift.yml

@@ -0,0 +1,15 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scylladb:aggregate-to-csi-external-provisioner-openshift
+  labels:
+    rbac.operator.scylladb.com/aggregate-to-csi-external-provisioner: "true"
+rules:
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - privileged
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use

k8s/scylla/05-4-scylla-csi-namespace.yml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-csi-driver

k8s/scylla/05-5-scylla-csi-storage-class.yml

@@ -0,0 +1,8 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: scylladb-local-xfs
+provisioner: local.csi.scylladb.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  csi.storage.k8s.io/fstype: xfs

k8s/scylla/05-6-scylla-csi-driver.yml

@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: local.csi.scylladb.com
+spec:
+  attachRequired: false
+  storageCapacity: true

k8s/scylla/05-7-scylla-csi-service-account.yml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: local-csi-driver
+  name: local-csi-driver

k8s/scylla/05-8-scylla-cs-cluster-role-binding.yml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scylladb:csi-external-provisioner
+subjects:
+- kind: ServiceAccount
+  name: local-csi-driver
+  namespace: local-csi-driver
+roleRef:
+  kind: ClusterRole
+  name: scylladb:csi-external-provisioner
+  apiGroup: rbac.authorization.k8s.io

k8s/scylla/05-9-scylla-csi-daemon-set.yml

@@ -0,0 +1,125 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  namespace: local-csi-driver
+  name: local-csi-driver
+  labels:
+    app.kubernetes.io/name: local-csi-driver
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: local-csi-driver
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: local-csi-driver
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+        scylla.scylladb.com/node-type: scylla
+      serviceAccountName: local-csi-driver
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: local-csi-driver
+        securityContext:
+          privileged: true
+        image: docker.io/scylladb/local-csi-driver:0.5.0@sha256:46c1d2e9be8edf076d8dea7e82a3bb763d4bdda5adf43c794f617e9c94251079
+        imagePullPolicy: IfNotPresent
+        args:
+        - --listen=/csi/csi.sock
+        - --node-name=$(NODE_NAME)
+        - --volumes-dir=/var/lib/persistent-volumes
+        - --v=2
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - name: kubelet-dir
+          mountPath: /var/lib/kubelet
+          mountPropagation: "Bidirectional"
+        - name: plugin-dir
+          mountPath: /csi
+        - name: volumes-dir
+          mountPath: /var/lib/persistent-volumes
+        ports:
+        - name: healthz
+          containerPort: 9809
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          timeoutSeconds: 3
+          periodSeconds: 2
+          failureThreshold: 5
+      - name: csi-driver-registrar
+        image: registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:fdff3ee285341bc58033b6b2458a5d45fd90ec6922a8ba6ebdd49b0c41e2cd34
+        imagePullPolicy: Always
+        args:
+        - --csi-address=/csi/csi.sock
+        - --kubelet-registration-path=/var/lib/kubelet/plugins/local.csi.scylladb.com/csi.sock
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+        - name: registration-dir
+          mountPath: /registration
+      - name: liveness-probe
+        image: registry.k8s.io/sig-storage/livenessprobe@sha256:cacee2b5c36dd59d4c7e8469c05c9e4ef53ecb2df9025fa8c10cdaf61bce62f0
+        imagePullPolicy: Always
+        args:
+        - --csi-address=/csi/csi.sock
+        - --health-port=9809
+        - --v=2
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+      - name: csi-provisioner
+        image: registry.k8s.io/sig-storage/csi-provisioner@sha256:ee3b525d5b89db99da3b8eb521d9cd90cb6e9ef0fbb651e98bb37be78d36b5b8
+        imagePullPolicy: Always
+        args:
+        - --csi-address=/csi/csi.sock
+        - --v=2
+        - --node-deployment
+        - --feature-gates=Topology=true
+        - --immediate-topology=false
+        - --enable-capacity
+        - --capacity-ownerref-level=0
+        - --capacity-poll-interval=30s
+        - --default-fstype=xfs
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+      volumes:
+      - name: kubelet-dir
+        hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+      - name: plugin-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins/local.csi.scylladb.com/
+          type: DirectoryOrCreate
+      - name: registration-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: Directory
+      - name: volumes-dir
+        hostPath:
+          path: /var/lib/persistent-volumes
+          type: Directory

k8s/scylla/06-scylla-manager.yml

@@ -0,0 +1,168 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: scylla-manager
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scylla-manager-config
+  namespace: scylla-manager
+data:
+  scylla-manager.yaml: |-
+    http: :5080
+    logger:
+      level: info
+    database:
+      hosts:
+      - scylla-manager-cluster-manager-dc-manager-rack-0
+
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  namespace: scylla-manager
+  name: scylla-manager-to-scylla-pod
+spec:
+  policyTypes:
+  - Ingress
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/managed-by: scylla-operator
+      app.kubernetes.io/name: scylla
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: scylla-manager
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: scylla-manager
+    app.kubernetes.io/instance: scylla-manager
+  name: scylla-manager
+  namespace: scylla-manager
+spec:
+  ports:
+  - name: api
+    port: 80
+    protocol: TCP
+    targetPort: 5080
+  - name: metrics
+    port: 5090
+    protocol: TCP
+    targetPort: 5090
+  selector:
+    app.kubernetes.io/name: scylla-manager
+    app.kubernetes.io/instance: scylla-manager
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scylla-manager
+  namespace: scylla-manager
+  labels:
+    app.kubernetes.io/name: scylla-manager
+    app.kubernetes.io/instance: scylla-manager
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scylla-manager
+  namespace: scylla-manager
+  labels:
+    app.kubernetes.io/name: scylla-manager
+    app.kubernetes.io/instance: scylla-manager
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: scylla-manager
+      app.kubernetes.io/instance: scylla-manager
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: scylla-manager
+        app.kubernetes.io/instance: scylla-manager
+    spec:
+      serviceAccountName: scylla-manager
+      containers:
+      - name: scylla-manager
+        image: docker.io/scylladb/scylla-manager:3.5.1@sha256:6986ecfc8c925c3d59b65bbcb9763d62f7591a00bb30242842aada115929e816
+        imagePullPolicy: IfNotPresent
+        command:
+        - /usr/bin/scylla-manager
+        args:
+        - --config-file=/mnt/etc/scylla-manager/scylla-manager.yaml
+        resources:
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        volumeMounts:
+        - mountPath: /mnt/etc/scylla-manager
+          name: scylla-manager-config
+        readinessProbe:
+          httpGet:
+            path: /api/v1/clusters
+            port: 5080
+          periodSeconds: 10
+          timeoutSeconds: 3
+      volumes:
+      - configMap:
+          name: scylla-manager-config
+        name: scylla-manager-config
+
+---
+apiVersion: scylla.scylladb.com/v1
+kind: ScyllaCluster
+metadata:
+  name: scylla-manager-cluster
+  namespace: scylla-manager
+spec:
+  version: 2025.1.5
+  agentVersion: 3.5.1@sha256:d1b57d08b9949c8faad2048fdf4dc7c502dae81da856c3c6b3a77dd347d5c7fc
+  repository: docker.io/scylladb/scylla
+  agentRepository: docker.io/scylladb/scylla-manager-agent
+  developerMode: true
+  cpuset: true
+  sysctls:
+    - fs.aio-max-nr=30000000
+  datacenter:
+    name: manager-dc
+    racks:
+    - members: 1
+      name: manager-rack
+      placement:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: scylla.scylladb.com/node-type
+                operator: In
+                values:
+                - scylla
+        tolerations:
+        - effect: NoSchedule
+          key: scylla-operator.scylladb.com/dedicated
+          operator: Equal
+          value: scyllaclusters
+      resources:
+        limits:
+          cpu: 1
+          memory: 200Mi
+        requests:
+          cpu: 1
+          memory: 200Mi
+      storage:
+        capacity: 5Gi
+        storageClassName: scylladb-local-xfs
+
+---

playbooks/dev/04-copy-k8s-files.yml

@@ -0,0 +1,18 @@
+- name: Copy K8s files to node
+  hosts: dev
+  become: true
+  vars:
+    k8s_files_src: "{{ playbook_dir }}/../../k8s/scylla"
+    k8s_files_dest: /tmp/k8s/scylla
+  tasks:
+    - name: Create destination directory for K8s files
+      ansible.builtin.file:
+        path: "{{ k8s_files_dest }}"
+        state: directory
+        mode: '0644'
+
+    - name: Copy K8s files to destination
+      ansible.builtin.copy:
+        src: "{{ k8s_files_src }}/"
+        dest: "{{ k8s_files_dest }}/"
+        mode: '0644'

playbooks/dev/05-cert-manager.yml

@@ -3,8 +3,6 @@
   become: true
   vars:
     cert_manager_ns: cert-manager
-    cert_manager_yaml_src: "{{ playbook_dir }}/../../k8s/scylla/01-cert-manager.yml"
-    cert_manager_yaml_dest: /tmp/01-cert-manager.yml
     cert_manager_crds:
       - certificates.cert-manager.io
       - issuers.cert-manager.io
@@ -13,16 +11,10 @@
       - cert-manager-cainjector
       - cert-manager-webhook
   tasks:
-    - name: Copy Cert-Manager YAML to node
-      ansible.builtin.copy:
-        src: "{{ cert_manager_yaml_src }}"
-        dest: "{{ cert_manager_yaml_dest }}"
-        mode: '0644'
-
     - name: Apply Cert-Manager YAML
       kubernetes.core.k8s:
         state: present
-        src: "{{ cert_manager_yaml_dest }}"
+        src: "/tmp/k8s/scylla/01-cert-manager.yml"
         kubeconfig: /etc/rancher/k3s/k3s.yaml
 
     - name: Wait for Cert-Manager CRDs

playbooks/dev/06-prometheus-operator.yml

@@ -2,21 +2,14 @@
   hosts: dev
   become: true
   vars:
-    prometheus_yaml_src: "{{ playbook_dir }}/../../k8s/scylla/02-prometheus-operator.yml"
-    prometheus_yaml_dest: /tmp/02-prometheus-operator.yml
     prometheus_crds:
       - prometheuses.monitoring.coreos.com
       - prometheusrules.monitoring.coreos.com
       - servicemonitors.monitoring.coreos.com
   tasks:
-    - name: Copy Prometheus YAML to node
-      ansible.builtin.copy:
-        src: "{{ prometheus_yaml_src }}"
-        dest: "{{ prometheus_yaml_dest }}"
-        mode: '0644'
-
     - name: Apply Prometheus operator server-side
-      ansible.builtin.command: kubectl apply --server-side -f {{ prometheus_yaml_dest }} -n prometheus-operator
+      ansible.builtin.command: |
+        kubectl apply --server-side -f /tmp/k8s/scylla/02-prometheus-operator.yml -n prometheus-operator
       environment:
         KUBECONFIG: /etc/rancher/k3s/k3s.yaml
       changed_when: false

playbooks/dev/06-scylla-operator.yml

@@ -1,41 +0,0 @@
-- name: Set up Scylla DB Operator
-  hosts: dev
-  become: true
-  vars:
-    scylla_yaml_src: "{{ playbook_dir }}/../../k8s/scylla/03-scylla-operator.yml"
-    scylla_yaml_dest: "/tmp/03-scylla-operator.yml"
-    scylla_crds:
-      - scyllaclusters.scylla.scylladb.com
-      - nodeconfigs.scylla.scylladb.com
-      - scyllaoperatorconfigs.scylla.scylladb.com
-      - scylladbmonitorings.scylla.scylladb.com
-    scylla_deployments:
-      - scylla-operator
-      - webhook-server
-  tasks:
-    - name: Copy Scylla Operator YAML to node
-      ansible.builtin.copy:
-        src: "{{ scylla_yaml_src }}"
-        dest: "{{ scylla_yaml_dest }}"
-        mode: '0644'
-
-    - name: Apply Scylla Operator YAML
-      ansible.builtin.command: kubectl apply --server-side -f {{ scylla_yaml_dest }} -n scylla-operator
-      environment:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-      changed_when: false
-
-    - name: Wait for Scylla Operator CRDs
-      ansible.builtin.shell: |
-        kubectl wait --for='condition=established' crd/{{ scylla_crds | join(' crd/') }}
-      changed_when: false
-
-    - name: Wait for Scylla Operator deployments
-      ansible.builtin.shell: |
-        kubectl -n scylla-operator rollout status --timeout=10m deployment.apps/{{ item }}
-      loop: "{{ scylla_deployments }}"
-      loop_control:
-        loop_var: item
-      environment:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-      changed_when: false

playbooks/dev/07-scylla-management.yml

@@ -0,0 +1,90 @@
+- name: Set up Scylla Operator, NodeConfig, Local CSI Driver, and Scylla Manager
+  hosts: dev
+  become: true
+  vars:
+    scylla_crds:
+      - scyllaclusters.scylla.scylladb.com
+      - nodeconfigs.scylla.scylladb.com
+      - scyllaoperatorconfigs.scylla.scylladb.com
+      - scylladbmonitorings.scylla.scylladb.com
+    scylla_deployments:
+      - scylla-operator
+      - webhook-server
+  tasks:
+    - name: Apply Scylla Operator YAML
+      ansible.builtin.command: |
+        kubectl apply --server-side -f /tmp/k8s/scylla/03-scylla-operator.yml -n scylla-operator
+      environment:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      changed_when: false
+
+    - name: Wait for Scylla Operator CRDs
+      ansible.builtin.shell: |
+        kubectl wait --for='condition=established' crd/{{ scylla_crds | join(' crd/') }}
+      changed_when: false
+
+    - name: Wait for Scylla Operator Deployments
+      ansible.builtin.shell: |
+        kubectl -n scylla-operator rollout status --timeout=10m deployment.apps/{{ item }}
+      loop: "{{ scylla_deployments }}"
+      loop_control:
+        loop_var: item
+      environment:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      changed_when: false
+
+    - name: Apply Scylla NodeConfig YAML
+      kubernetes.core.k8s:
+        state: present
+        src: "/tmp/k8s/scylla/04-scylla-node-config.yml"
+        namespace: scylla-operator
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+
+    - name: Wait for Scylla NodeConfig to be applied across all nodes
+      ansible.builtin.command: |
+        kubectl wait --for='condition=Reconciled' --timeout=10m nodeconfigs.scylla.scylladb.com/scylladb-nodepool-1
+      changed_when: false
+
+    - name: Find Scylla CSI Driver K8s YAMLs on Remote Host
+      ansible.builtin.find:
+        paths: /tmp/k8s/scylla
+        patterns: "05-*.yml"
+        file_type: file
+      register: scylla_csi_driver_files
+
+    - name: Apply all Scylla CSI Driver K8s YAMLs
+      kubernetes.core.k8s:
+        state: present
+        src: "{{ item.path }}"
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+      loop: "{{ scylla_csi_driver_files.files | sort(attribute='path') }}"
+      loop_control:
+        label: "{{ item.path }}"
+
+    - name: Wait for Local CSI Driver DaemonSet
+      ansible.builtin.command: |
+        kubectl -n=local-csi-driver rollout status --timeout=10m daemonset.apps/local-csi-driver
+      changed_when: false
+
+    - name: Apply Scylla Manager YAML
+      kubernetes.core.k8s:
+        state: present
+        src: "/tmp/k8s/scylla/06-scylla-manager.yml"
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+
+    - name: Label current node as Scylla
+      kubernetes.core.k8s:
+        api_version: v1
+        kind: Node
+        name: "{{ inventory_hostname }}"
+        definition:
+          metadata:
+            labels:
+              scylla.scylladb.com/node-type: scylla
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        state: present
+
+    - name: Wait for Scylla Manager Deployment
+      ansible.builtin.command: |
+        kubectl -n=scylla-manager rollout status --timeout=10m deployment.apps/scylla-manager
+      changed_when: false