feat: 🎉 Initial commit to Nyeki's Git.

2024-10-31 00:42:52 -03:00
parent 345e44137a
commit d20f808b88
7 changed files with 184 additions and 127 deletions
@@ -1,8 +1,136 @@
 # Payphone

-Payphone is deployed using Kubernetes. To set up a local kubernetes cluster, you can use `minikube`.
+Payphone is deployed using Kubernetes + Docker. To set up a local kubernetes cluster, you can use
+`minikube`.
+
+## Starting a Cluster
+
+This section will guide you step by step to create a multi-node cluster or a local development
+cluster and start up Payphone. Make sure to have:
+
+1. A Docker Hub account with a slot for a private image. By default all free accounts allow 1
+   private image, so if you haven't uploaded any private images to your account, you do have an
+   available slot.
+2. The Discord bot's token, either for development or production depending on what you're setting
+   up the cluster for.
+3. You'll probably need at least 5 CPU cores (6 recommended) across all of your machines + 1 core
+   per shard, 8 GB of ram + 512Mb of RAM per shard, and 30Gb of disk space available. It'll still
+   run with less, but it'll be way slower. In multi-node environment, these specifications are
+   cluster-wide, meaning that if you have 3 nodes with 3 cores, that makes it 9 cores in total and
+   therefore suitable for a multi-shard deployment. Do not run nodes with less than 2 CPU cores, 2
+   Gb (ideally 4 Gb) of RAM, and 15 Gb of disk space available.
+
+### Kubernetes Setup
+
+First of all, you need to install Kubernetes in the machine where you'll set up either a node or a
+whole cluster.
+
+Make sure to have Docker installed in your machine before running the following steps. You can
+check how to install Docker in the
+[Docker documentation](https://docs.docker.com/get-started/get-docker/). If you're setting up a new
+node for production, it's better if you only install the Docker Engine without Docker Desktop.
+
+#### For Development
+
+Ideally, use `minikube` for local development. This'll allow you to run a complete cluster locally.
+Make sure to have downloaded and installed minikube first. On Windows, it's available with
+`chocolatey` running `choco install minikube`. You can read the full set up guide for other
+installation methods at the [official minikube documentation](https://minikube.sigs.k8s.io/docs/).
+
+After `minikube` is installed, you need to set up a local cluster. This'll create a virtual machine
+in a Docker container to act as nodes in your cluster. Run the following command to start a 3-node
+local cluster:

 ```sh
-# To create a local cluster of 3 nodes:
 minikube start --network-plugin=cni --cni=calico --nodes=3
 ```
+
+This'll create 3 docker containers and start them, and set them up to use kubectl connecting them
+automatically. Management `kubectl` commands can only be ran from the master/control-panel node,
+but with `kubectl` you don't need to worry about that since running `kubectl` from your machine
+will work as if you were in a master node.
+
+`kubectl` comes integrated with `Docker Desktop`. You can enable Kubernetes in Docker Desktop at
+Docker Desktop > Settings > Kubernetes > Enable Kubernetes.
+
+#### For Production
+
+These steps are required in order to set up a new `microk8s` node. These installation steps assume
+they're being ran in an Ubuntu 22.4+ server with at least 2 CPU cores, 4 GB RAM, and 25 GB disk
+space, and `snap` installed. You also need root privileges to follow these steps.
+
+First of all, install `microk8s`.
+
+```sh
+sudo snap install microk8s --classic
+```
+
+After the installation, check that `microk8s` is up and running using the following command:
+
+```sh
+microk8s status --wait-ready
+```
+
+> [!INFO] In case the last command fails to run because of missing permissions, you'll be guided
+> on how to give your current user the required permissions to run the command. Follow those steps
+> and re-run the command afterwards.
+
+Then, you need to enable the required extensions for `microk8s` to work properly. Run the following
+commands to enable them:
+
+```sh
+microk8s enable dns
+microk8s enable ha-cluster
+```
+
+The rest of the guide uses `kubectl`, but `microk8s` has `kubectl` as a subcommand, so either:
+
+1. You create an alias in `.bash_aliases`, appending `alias kubectl='microk8s kubectl'` to your
+   `~/.bash_aliases` file. If you don't have one, create an empty file and add the command in it.
+   After updating the file, run `source ~/.bash_aliases` to apply the changes.
+
+2. Use `microk8s kubectl` instead of `kubectl` in the following steps. This'll require you to type
+   `microk8s` at the front of all `kubectl` commands that are run from now on.
+
+## Deploying Payphone
+
+The following steps show how to pull and install Payphone on a running cluster of nodes.
+
+### Building and Publishing the Payphone Image
+
+To be able to deploy Payphone to your k8s cluster, you first need to upload the image to a private
+registry. This documentation uses Docker Hub as a registry
+
+```sh
+# Only run this step if you need to build the source code and it is in your machine. Build and
+# publish to the private registry. Use your own image name, for example `samantha/payphone:latest`.
+docker login
+
+docker build -t nekidev/payphone:latest .
+docker publish nekidev/payphone:latest
+
+# In the following steps, if you're using microk8s in production, use `microk8s kubectl` instead of
+# `kubectl`. You can add this to your ~/.bash_aliases to avoid having to prepend `microk8s` to all
+# commands:
+# alias kubectl='microk8s kubectl'
+
+# This step requires you to create a personal access token for Docker Hub. To create it:
+# 1. Go to https://app.docker.com/settings/personal-access-tokens/create
+# 2. Set the access permission to "Read-only"
+kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<your-username> --docker-password=<your-personal-access-token>
+
+
+# This will pull the images for citus and redis and start their containers.
+kubectl apply -f redis.yml
+kubectl apply -f citus.yml
+
+# After applying citus.yml, you'll need to connect the nodes. We'll always use citus-0 as the
+# master instance.
+
+# Steps before running this command:
+# 1. Update the image name in `bot.yml` to your own image name (the one that you have in your
+#      private docker hub registry)
+# 2. Set the total number of shards in both `spec.replicas` and
+#      `spec.template.spec.containers.bot.env.TOTAL_SHARDS.value`.
+kubectl apply -f bot.yml
+```
@@ -1,18 +1,19 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
-  name: discord-bot
+  name: bot
  labels:
-    app: discord-bot
+    app: bot
 spec:
-  replicas: 11
+  serviceName: "bot-headless"
+  replicas: 2
  selector:
    matchLabels:
-      app: discord-bot
+      app: bot
  template:
    metadata:
      labels:
-        app: discord-bot
+        app: bot
    spec:
      containers:
        - name: bot
@@ -20,16 +21,12 @@ spec:
          env:
            - name: IS_DEBUG
              value: "false"
-            - name: SHARD_NAME
+            - name: SHARD_ID
              valueFrom:
                fieldRef:
-                  fieldPath: metadata.name
+                  fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
            - name: TOTAL_SHARDS
-              value: "11"
-            - name: REDIS_HOST
-              value: "redis"
-            - name: REDIS_PORT
-              value: "6379"
+              value: "2"  # Adjust this manually or with a ConfigMap
          resources:
            limits:
              cpu: "500m"
@@ -37,4 +34,25 @@ spec:
            requests:
              cpu: "250m"
              memory: "256Mi"
-          restartPolicy: Always
+      imagePullSecrets:
+        - name: regcred
+  volumeClaimTemplates:
+    - metadata:
+        name: bot-storage
+      spec:
+        accessModes:
+          - ReadWriteOnce  # Ensures only one node can mount the volume at a time
+        resources:
+          requests:
+            storage: 1Gi  # Defines the storage size
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: bot-headless
+  labels:
+    app: bot
+spec:
+  clusterIP: None
+  selector:
+    app: bot
@@ -1,57 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: citus-coordinator
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: citus
-      role: coordinator
-  template:
-    metadata:
-      labels:
-        app: citus
-        role: coordinator
-    spec:
-      containers:
-        - name: coordinator
-          image: citusdata/citus:latest
-          env:
-            - name: POSTGRES_DB
-              value: "payphone"
-            - name: POSTGRES_USER
-              value: "payphone"
-            - name: POSTGRES_PASSWORD
-              value: "payphone"
-          volumeMounts:
-            - name: citus-coordinator-storage
-              mountPath: /var/lib/postgresql/data
-          ports:
-            - containerPort: 5432
-          resources:
-            requests:
-              memory: "2Gi"
-              cpu: "1"
-            limits:
-              memory: "2Gi"
-              cpu: "2"
-      volumes:
-        - name: citus-coordinator-storage
-          persistentVolumeClaim:
-            claimName: citus-coordinator-pvc
-
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: citus-coordinator
-spec:
-  selector:
-    app: citus
-    role: coordinator
-  ports:
-    - protocol: TCP
-      port: 5432
-      targetPort: 5432
-  type: ClusterIP
@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: citus-coordinator-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi  # Change this size as needed
-
---
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: citus-worker-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi  # Change this size as needed
@@ -1,22 +1,34 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: citus-headless
+spec:
+  selector:
+    app: citus
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+  clusterIP: None  # Headless service for direct pod access
+
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: citus-worker
+  name: citus
 spec:
-  replicas: 2  # Increase as needed for more workers
+  serviceName: citus-headless
+  replicas: 3  # Set the desired number of replicas
  selector:
    matchLabels:
      app: citus
-      role: worker
-  serviceName: citus-worker-headless
  template:
    metadata:
      labels:
        app: citus
-        role: worker
    spec:
      containers:
-        - name: worker
+        - name: citus
          image: citusdata/citus:latest
          env:
            - name: POSTGRES_DB
@@ -27,37 +39,15 @@ spec:
              value: "payphone"
          ports:
            - containerPort: 5432
-          resources:
-            requests:
-              memory: "2Gi"
-              cpu: "1"
-            limits:
-              memory: "2Gi"
-              cpu: "2"
          volumeMounts:
-            - name: citus-worker-storage
+            - name: citus-storage
              mountPath: /var/lib/postgresql/data
  volumeClaimTemplates:
    - metadata:
-        name: citus-worker-storage
+        name: citus-storage
      spec:
        accessModes: [ "ReadWriteOnce" ]
        storageClassName: "standard"  # Update based on your storage class
        resources:
          requests:
-            storage: 5Gi  # Size per worker
-
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: citus-worker-headless
-spec:
-  selector:
-    app: citus
-    role: worker
-  ports:
-    - protocol: TCP
-      port: 5432
-      targetPort: 5432
-  clusterIP: None  # Defines the service as headless, allowing direct pod DNS entries
+            storage: 5Gi  # Size for each pod's storage