Update a boatload of ansible stuff

inventory-dev.ini

@@ -0,0 +1,8 @@
+[init]
+p4 ansible_host=p4.vps.nyeki.dev ansible_user=root rack=ge-dc1-rack1
+
+[master]
+p5 ansible_host=p5.vps.nyeki.dev ansible_user=root rack=ge-dc1-rack2
+
+[worker]
+p6 ansible_host=p6.vps.nyeki.dev ansible_user=root rack=ge-dc1-rack3

inventory.ini

@@ -1,2 +0,0 @@
-[dev]
-p4 ansible_host=p4.vps.nyeki.dev ansible_user=root

k8s/ansible/argo-cd/02-scylla-app.yml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: scylla
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: scylla
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: 'https://git.nyeki.dev/payphone/payphone.git'
+    targetRevision: rewrite
+    path: k8s/argo-cd/scylla
+  destination:
+    server: 'https://kubernetes.default.svc'
+    namespace: scylla
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argo-cd/scylla/01-scylla-cluster.yml

@@ -0,0 +1,117 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scylladb-config
+data:
+  scylla.yaml: |
+    authenticator: PasswordAuthenticator
+    authorizer: CassandraAuthorizer
+---
+apiVersion: scylla.scylladb.com/v1
+kind: ScyllaCluster
+metadata:
+  name: scylladb
+spec:
+  repository: docker.io/scylladb/scylla
+  version: 2025.1.5
+  agentVersion: 3.5.1
+  developerMode: false
+  automaticOrphanedNodeCleanup: true
+  sysctls:
+  - fs.aio-max-nr=30000000
+  datacenter:
+    name: ge-dc1
+  racks:
+  - name: ge-dc1-rack1
+    members: 1
+    scyllaConfig: scylladb-config
+    storage:
+      capacity: 50Gi
+      storageClassName: scylladb-local-xfs
+    resources:
+      requests:
+        cpu: 1
+        memory: 8Gi
+      limits:
+        cpu: 1
+        memory: 8Gi
+    placement:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: topology.kubernetes.io/zone
+              operator: In
+              values:
+              - ge-dc1-rack1
+            - key: scylla.scylladb.com/node-type
+              operator: In
+              values:
+              - scylla
+      tolerations:
+      - key: scylla-operator.scylladb.com/dedicated
+        operator: Equal
+        value: scyllaclusters
+        effect: NoSchedule
+  - name: ge-dc1-rack2
+    members: 1
+    scyllaConfig: scylladb-config
+    storage:
+      capacity: 50Gi
+      storageClassName: scylladb-local-xfs
+    resources:
+      requests:
+        cpu: 1
+        memory: 8Gi
+      limits:
+        cpu: 1
+        memory: 8Gi
+    placement:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: topology.kubernetes.io/zone
+              operator: In
+              values:
+              - ge-dc1-rack2
+            - key: scylla.scylladb.com/node-type
+              operator: In
+              values:
+              - scylla
+      tolerations:
+      - key: scylla-operator.scylladb.com/dedicated
+        operator: Equal
+        value: scyllaclusters
+        effect: NoSchedule
+  - name: ge-dc1-rack3
+    members: 1
+    scyllaConfig: scylladb-config
+    storage:
+      capacity: 50Gi
+      storageClassName: scylladb-local-xfs
+    resources:
+      requests:
+        cpu: 1
+        memory: 8Gi
+      limits:
+        cpu: 1
+        memory: 8Gi
+    placement:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: topology.kubernetes.io/zone
+              operator: In
+              values:
+              - ge-dc1-rack3
+            - key: scylla.scylladb.com/node-type
+              operator: In
+              values:
+              - scylla
+      tolerations:
+      - key: scylla-operator.scylladb.com/dedicated
+        operator: Equal
+        value: scyllaclusters
+        effect: NoSchedule

playbooks/00-run-dev.sh

@@ -0,0 +1,9 @@
+uv run ansible-playbook -i inventory-dev.ini playbooks/01-setup.yml
+uv run ansible-playbook -i inventory-dev.ini playbooks/02-copy-k8s-files.yml
+uv run ansible-playbook -i inventory-dev.ini playbooks/03-argo-cd.yml
+uv run ansible-playbook -i inventory-dev.ini playbooks/04-secrets.yml --ask-vault-pass
+uv run ansible-playbook -i inventory-dev.ini playbooks/05-cert-manager.yml
+uv run ansible-playbook -i inventory-dev.ini playbooks/06-prometheus-operator.yml
+uv run ansible-playbook -i inventory-dev.ini playbooks/07-scylla-operator.yml
+uv run ansible-playbook -i inventory-dev.ini playbooks/08-scylla-cluster.yml
+uv run ansible-playbook -i inventory-dev.ini playbooks/09-scylla-manager.yml

playbooks/01-setup.yml

@@ -0,0 +1,116 @@
+- name: Download K3s Setup Script and K8s Python Lib on All Nodes
+  hosts: all
+  become: true
+  strategy: linear
+  tasks:
+    - name: Install curl
+      ansible.builtin.apt:
+        name: curl
+        state: present
+      when: ansible_os_family == "Debian"
+
+    - name: Download k3s install script
+      ansible.builtin.get_url:
+        url: https://get.k3s.io
+        dest: /tmp/k3s_install.sh
+        mode: '0755'
+
+    - name: Install Python Kubernetes client
+      ansible.builtin.apt:
+        name: python3-kubernetes
+        state: present
+
+    - name: Disable UFW
+      ansible.builtin.command: ufw disable
+      changed_when: false
+
+- name: Set up K3s on Init Node
+  hosts: init
+  become: true
+  strategy: linear
+  tasks:
+    - name: Run k3s Installer
+      ansible.builtin.shell: |
+        INSTALL_K3S_EXEC="server --tls-san {{ ansible_host }} --flannel-backend=wireguard-native --cluster-init" \
+        sh /tmp/k3s_install.sh
+      args:
+        creates: /usr/local/bin/k3s
+
+    - name: Ensure K3s Service is Started and Enabled
+      ansible.builtin.systemd:
+        name: k3s
+        enabled: true
+        state: started
+
+    - name: Get Joining Token
+      ansible.builtin.command: cat /var/lib/rancher/k3s/server/node-token
+      register: k3s_token
+      run_once: true
+      tags: skip_ansible_lint
+      changed_when: false
+
+- name: Set up K3s on Master Nodes
+  hosts: master
+  become: true
+  strategy: linear
+  tasks:
+    - name: Run k3s Installer
+      ansible.builtin.shell: |
+        INSTALL_K3S_EXEC="server --flannel-backend=wireguard-native --tls-san {{ ansible_host }}" \
+        K3S_URL="https://{{ hostvars[groups['init'][0]].ansible_host }}:6443" \
+        K3S_TOKEN="{{ hostvars[groups['init'][0]].k3s_token.stdout }}" sh /tmp/k3s_install.sh
+      args:
+        creates: /usr/local/bin/k3s
+
+    - name: Ensure k3s service is started and enabled
+      ansible.builtin.systemd:
+        name: k3s
+        enabled: true
+        state: started
+
+- name: Set up K3s on Worker Nodes
+  hosts: worker
+  become: true
+  strategy: linear
+  tasks:
+    - name: Run k3s Installer
+      ansible.builtin.shell: |
+        INSTALL_K3S_EXEC="agent" K3S_URL="https://{{ hostvars[groups['init'][0]].ansible_host }}:6443" \
+        K3S_TOKEN="{{ hostvars[groups['init'][0]].k3s_token.stdout }}" sh /tmp/k3s_install.sh
+      args:
+        creates: /usr/local/bin/k3s
+
+    - name: Ensure k3s service is started and enabled
+      ansible.builtin.systemd:
+        name: k3s-agent
+        enabled: true
+        state: started
+
+- name: Install Helm in All Nodes
+  hosts: all
+  become: true
+  strategy: linear
+  tasks:
+    - name: Download Helm Installation Script
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+        dest: /tmp/get_helm.sh
+        mode: '0755'
+
+    - name: Run helm installer
+      ansible.builtin.command: /tmp/get_helm.sh
+      args:
+        creates: /usr/local/bin/helm
+
+- name: Wait for All K3s Nodes to Become Ready
+  hosts: init
+  become: true
+  tasks:
+    - name: Wait until all nodes report Ready
+      ansible.builtin.shell: |
+        kubectl wait --for=condition=Ready node --all --timeout=10m
+      register: wait_nodes
+      retries: 5
+      delay: 30
+      until: wait_nodes.rc == 0
+      changed_when: false

playbooks/02-copy-k8s-files.yml

@@ -1,9 +1,10 @@
 - name: Copy K8s files to node
-  hosts: dev
+  hosts: init
   become: true
+  gather_facts: false
   vars:
-    k8s_files_src: "{{ playbook_dir }}/../../k8s/scylla"
+    k8s_files_src: "{{ playbook_dir }}/../k8s"
-    k8s_files_dest: /tmp/k8s/scylla
+    k8s_files_dest: /tmp/k8s
   tasks:
     - name: Create destination directory for K8s files
       ansible.builtin.file:
@@ -12,7 +13,7 @@
         mode: '0644'
 
     - name: Copy K8s files to destination
-      ansible.builtin.copy:
+      ansible.posix.synchronize:
         src: "{{ k8s_files_src }}/"
         dest: "{{ k8s_files_dest }}/"
-        mode: '0644'
+        recursive: true

playbooks/03-argo-cd.yml

@@ -0,0 +1,17 @@
+- name: Set up ArgoCD
+  hosts: init
+  become: true
+  tasks:
+    - name: Create ArgoCD namespace
+      kubernetes.core.k8s:
+        api_version: v1
+        kind: Namespace
+        name: argocd
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+
+    - name: Apply ArgoCD installation manifest
+      kubernetes.core.k8s:
+        state: present
+        src: /tmp/k8s/ansible/argo-cd/01-argo-cd.yml
+        namespace: argocd
+        kubeconfig: /etc/rancher/k3s/k3s.yaml

playbooks/04-secrets.yml

@@ -1,8 +1,9 @@
 - name: Set up repository secrets
-  hosts: dev
+  hosts: init
   become: true
+  gather_facts: false
   vars_files:
-    - ../secrets.yml
+    - 00-secrets.yml
   tasks:
     - name: Add private Git repo credentials to ArgoCD
       kubernetes.core.k8s:
@@ -13,9 +14,11 @@
           metadata:
             name: payphone-repo-creds
             namespace: argocd
-            annotations:
+            labels:
               argocd.argoproj.io/secret-type: repository
+          type: Opaque
           stringData:
+            type: git
             url: https://git.nyeki.dev/payphone/payphone.git
             username: "{{ gitlab_username }}"
             password: "{{ gitlab_pat }}"

playbooks/05-cert-manager.yml

@@ -1,5 +1,5 @@
 - name: Set up Cert-Manager
-  hosts: dev
+  hosts: init
   become: true
   vars:
     cert_manager_ns: cert-manager
@@ -14,7 +14,7 @@
     - name: Apply Cert-Manager YAML
       kubernetes.core.k8s:
         state: present
-        src: "/tmp/k8s/scylla/01-cert-manager.yml"
+        src: "/tmp/k8s/ansible/scylla/01-cert-manager.yml"
         kubeconfig: /etc/rancher/k3s/k3s.yaml
 
     - name: Wait for Cert-Manager CRDs

playbooks/06-prometheus-operator.yml

@@ -1,5 +1,5 @@
 - name: Set up Prometheus operator
-  hosts: dev
+  hosts: init
   become: true
   vars:
     prometheus_crds:
@@ -9,7 +9,7 @@
   tasks:
     - name: Apply Prometheus operator server-side
       ansible.builtin.command: |
-        kubectl apply --server-side -f /tmp/k8s/scylla/02-prometheus-operator.yml -n prometheus-operator
+        kubectl apply --server-side -f /tmp/k8s/ansible/scylla/02-prometheus-operator.yml -n prometheus-operator
       environment:
         KUBECONFIG: /etc/rancher/k3s/k3s.yaml
       changed_when: false

playbooks/07-scylla-operator.yml

@@ -1,5 +1,5 @@
-- name: Set up Scylla Operator, NodeConfig, Local CSI Driver, and Scylla Manager
+- name: Set up Scylla Operator, NodeConfig, and the Local CSI Driver
-  hosts: dev
+  hosts: init
   become: true
   vars:
     scylla_crds:
@@ -13,7 +13,7 @@
   tasks:
     - name: Apply Scylla Operator YAML
       ansible.builtin.command: |
-        kubectl apply --server-side -f /tmp/k8s/scylla/03-scylla-operator.yml -n scylla-operator
+        kubectl apply --server-side -f /tmp/k8s/ansible/scylla/03-scylla-operator.yml -n scylla-operator
       environment:
         KUBECONFIG: /etc/rancher/k3s/k3s.yaml
       changed_when: false
@@ -36,7 +36,7 @@
     - name: Apply Scylla NodeConfig YAML
       kubernetes.core.k8s:
         state: present
-        src: "/tmp/k8s/scylla/04-scylla-node-config.yml"
+        src: "/tmp/k8s/ansible/scylla/04-scylla-node-config.yml"
         namespace: scylla-operator
         kubeconfig: /etc/rancher/k3s/k3s.yaml
 
@@ -66,25 +66,15 @@
         kubectl -n=local-csi-driver rollout status --timeout=10m daemonset.apps/local-csi-driver
       changed_when: false
 
-    - name: Apply Scylla Manager YAML
-      kubernetes.core.k8s:
-        state: present
-        src: "/tmp/k8s/scylla/06-scylla-manager.yml"
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-
     - name: Label current node as Scylla
+      loop: groups['all']
       kubernetes.core.k8s:
         api_version: v1
         kind: Node
-        name: "{{ inventory_hostname }}"
+        name: "{{ item }}"
         definition:
           metadata:
             labels:
               scylla.scylladb.com/node-type: scylla
         kubeconfig: /etc/rancher/k3s/k3s.yaml
         state: present
-
-    - name: Wait for Scylla Manager Deployment
-      ansible.builtin.command: |
-        kubectl -n=scylla-manager rollout status --timeout=10m deployment.apps/scylla-manager
-      changed_when: false

playbooks/08-scylla-cluster.yml

@@ -0,0 +1,20 @@
+- name: Set Up Scylla Cluster
+  hosts: init
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Set Up ArgoCD Application for Scylla Cluster
+      kubernetes.core.k8s:
+        state: present
+        src: /tmp/k8s/ansible/argo-cd/02-scylla-app.yml
+        namespace: argocd
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+
+    - name: Wait for Scylla Cluster Application
+      ansible.builtin.command: |
+        kubectl -n scylla rollout status statefulset/{{ item }} --timeout=10m
+      loop:
+        - ge-dc1-rack1
+        - ge-dc1-rack2
+        - ge-dc1-rack3
+      changed_when: false

playbooks/09-scylla-manager.yml

@@ -0,0 +1,14 @@
+- name: Set Up Scylla Manager
+  hosts: init
+  become: true
+  tasks:
+    - name: Apply Scylla Manager YAML
+      kubernetes.core.k8s:
+        state: present
+        src: "/tmp/k8s/ansible/scylla/06-scylla-manager.yml"
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+
+    - name: Wait for Scylla Manager Deployment
+      ansible.builtin.command: |
+        kubectl -n=scylla-manager rollout status --timeout=10m deployment.apps/scylla-manager
+      changed_when: false

playbooks/dev/01-setup.yml

@@ -1,42 +0,0 @@
-- name: Set up k3s on dev node
-  hosts: dev
-  become: true
-  tasks:
-    - name: Install curl
-      ansible.builtin.apt:
-        name: curl
-        state: present
-      when: ansible_os_family == "Debian"
-
-    - name: Download k3s install script
-      ansible.builtin.get_url:
-        url: https://get.k3s.io
-        dest: /tmp/k3s_install.sh
-        mode: '0755'
-
-    - name: Run k3s installer
-      ansible.builtin.command: /tmp/k3s_install.sh
-      args:
-        creates: /usr/local/bin/k3s
-
-    - name: Ensure k3s service is started and enabled
-      ansible.builtin.systemd:
-        name: k3s
-        enabled: true
-        state: started
-
-    - name: Install Python Kubernetes client
-      ansible.builtin.apt:
-        name: python3-kubernetes
-        state: present
-
-    - name: Download helm install script
-      ansible.builtin.get_url:
-        url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
-        dest: /tmp/get_helm.sh
-        mode: '0755'
-
-    - name: Run helm installer
-      ansible.builtin.command: /tmp/get_helm.sh
-      args:
-        creates: /usr/local/bin/helm

playbooks/dev/02-argocd.yml

@@ -1,30 +0,0 @@
-- name: Set up ArgoCD
-  hosts: dev
-  become: true
-  tasks:
-    - name: Create ArgoCD namespace
-      kubernetes.core.k8s:
-        api_version: v1
-        kind: Namespace
-        name: argocd
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-
-    - name: Apply ArgoCD installation manifest
-      kubernetes.core.k8s:
-        state: present
-        src: https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
-        namespace: argocd
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-
-    - name: Get ArgoCD initial admin password
-      ansible.builtin.shell: kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}"
-      register: encoded_pass
-      changed_when: false
-
-    - name: Decode ArgoCD password
-      ansible.builtin.set_fact:
-        argocd_pass: "{{ encoded_pass.stdout | b64decode }}"
-
-    - name: Show ArgoCD password
-      ansible.builtin.debug:
-        msg: "ArgoCD admin password: {{ argocd_pass }}"