diff --git a/README.md b/README.md index d3b44be..f2459ec 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,136 @@ # Payphone -Payphone is deployed using Kubernetes. To set up a local kubernetes cluster, you can use `minikube`. +Payphone is deployed using Kubernetes + Docker. To set up a local kubernetes cluster, you can use +`minikube`. + +## Starting a Cluster + +This section will guide you step by step to create a multi-node cluster or a local development +cluster and start up Payphone. Make sure to have: + +1. A Docker Hub account with a slot for a private image. By default all free accounts allow 1 + private image, so if you haven't uploaded any private images to your account, you do have an + available slot. +2. The Discord bot's token, either for development or production depending on what you're setting + up the cluster for. +3. You'll probably need at least 5 CPU cores (6 recommended) across all of your machines + 1 core + per shard, 8 GB of ram + 512Mb of RAM per shard, and 30Gb of disk space available. It'll still + run with less, but it'll be way slower. In multi-node environment, these specifications are + cluster-wide, meaning that if you have 3 nodes with 3 cores, that makes it 9 cores in total and + therefore suitable for a multi-shard deployment. Do not run nodes with less than 2 CPU cores, 2 + Gb (ideally 4 Gb) of RAM, and 15 Gb of disk space available. + +### Kubernetes Setup + +First of all, you need to install Kubernetes in the machine where you'll set up either a node or a +whole cluster. + +Make sure to have Docker installed in your machine before running the following steps. You can +check how to install Docker in the +[Docker documentation](https://docs.docker.com/get-started/get-docker/). If you're setting up a new +node for production, it's better if you only install the Docker Engine without Docker Desktop. + +#### For Development + +Ideally, use `minikube` for local development. This'll allow you to run a complete cluster locally. +Make sure to have downloaded and installed minikube first. On Windows, it's available with +`chocolatey` running `choco install minikube`. You can read the full set up guide for other +installation methods at the [official minikube documentation](https://minikube.sigs.k8s.io/docs/). + +After `minikube` is installed, you need to set up a local cluster. This'll create a virtual machine +in a Docker container to act as nodes in your cluster. Run the following command to start a 3-node +local cluster: ```sh -# To create a local cluster of 3 nodes: minikube start --network-plugin=cni --cni=calico --nodes=3 -``` \ No newline at end of file +``` + +This'll create 3 docker containers and start them, and set them up to use kubectl connecting them +automatically. Management `kubectl` commands can only be ran from the master/control-panel node, +but with `kubectl` you don't need to worry about that since running `kubectl` from your machine +will work as if you were in a master node. + +`kubectl` comes integrated with `Docker Desktop`. You can enable Kubernetes in Docker Desktop at +Docker Desktop > Settings > Kubernetes > Enable Kubernetes. + +#### For Production + +These steps are required in order to set up a new `microk8s` node. These installation steps assume +they're being ran in an Ubuntu 22.4+ server with at least 2 CPU cores, 4 GB RAM, and 25 GB disk +space, and `snap` installed. You also need root privileges to follow these steps. + +First of all, install `microk8s`. + +```sh +sudo snap install microk8s --classic +``` + +After the installation, check that `microk8s` is up and running using the following command: + +```sh +microk8s status --wait-ready +``` + +> [!INFO] In case the last command fails to run because of missing permissions, you'll be guided +> on how to give your current user the required permissions to run the command. Follow those steps +> and re-run the command afterwards. + +Then, you need to enable the required extensions for `microk8s` to work properly. Run the following +commands to enable them: + +```sh +microk8s enable dns +microk8s enable ha-cluster +``` + +The rest of the guide uses `kubectl`, but `microk8s` has `kubectl` as a subcommand, so either: + +1. You create an alias in `.bash_aliases`, appending `alias kubectl='microk8s kubectl'` to your + `~/.bash_aliases` file. If you don't have one, create an empty file and add the command in it. + After updating the file, run `source ~/.bash_aliases` to apply the changes. + +2. Use `microk8s kubectl` instead of `kubectl` in the following steps. This'll require you to type + `microk8s` at the front of all `kubectl` commands that are run from now on. + +## Deploying Payphone + +The following steps show how to pull and install Payphone on a running cluster of nodes. + +### Building and Publishing the Payphone Image + +To be able to deploy Payphone to your k8s cluster, you first need to upload the image to a private +registry. This documentation uses Docker Hub as a registry + +```sh +# Only run this step if you need to build the source code and it is in your machine. Build and +# publish to the private registry. Use your own image name, for example `samantha/payphone:latest`. +docker login + +docker build -t nekidev/payphone:latest . +docker publish nekidev/payphone:latest + +# In the following steps, if you're using microk8s in production, use `microk8s kubectl` instead of +# `kubectl`. You can add this to your ~/.bash_aliases to avoid having to prepend `microk8s` to all +# commands: +# alias kubectl='microk8s kubectl' + +# This step requires you to create a personal access token for Docker Hub. To create it: +# 1. Go to https://app.docker.com/settings/personal-access-tokens/create +# 2. Set the access permission to "Read-only" +kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username= --docker-password= + + +# This will pull the images for citus and redis and start their containers. +kubectl apply -f redis.yml +kubectl apply -f citus.yml + +# After applying citus.yml, you'll need to connect the nodes. We'll always use citus-0 as the +# master instance. + +# Steps before running this command: +# 1. Update the image name in `bot.yml` to your own image name (the one that you have in your +# private docker hub registry) +# 2. Set the total number of shards in both `spec.replicas` and +# `spec.template.spec.containers.bot.env.TOTAL_SHARDS.value`. +kubectl apply -f bot.yml +``` diff --git a/assets/avatar.png b/assets/avatar.png new file mode 100644 index 0000000..2ce5967 Binary files /dev/null and b/assets/avatar.png differ diff --git a/assets/banner.png b/assets/banner.png new file mode 100644 index 0000000..2a91047 Binary files /dev/null and b/assets/banner.png differ diff --git a/k8s/bot.yml b/k8s/bot.yml index ca266ec..1b840be 100644 --- a/k8s/bot.yml +++ b/k8s/bot.yml @@ -1,18 +1,19 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: discord-bot + name: bot labels: - app: discord-bot + app: bot spec: - replicas: 11 + serviceName: "bot-headless" + replicas: 2 selector: matchLabels: - app: discord-bot + app: bot template: metadata: labels: - app: discord-bot + app: bot spec: containers: - name: bot @@ -20,16 +21,12 @@ spec: env: - name: IS_DEBUG value: "false" - - name: SHARD_NAME + - name: SHARD_ID valueFrom: fieldRef: - fieldPath: metadata.name + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] - name: TOTAL_SHARDS - value: "11" - - name: REDIS_HOST - value: "redis" - - name: REDIS_PORT - value: "6379" + value: "2" # Adjust this manually or with a ConfigMap resources: limits: cpu: "500m" @@ -37,4 +34,25 @@ spec: requests: cpu: "250m" memory: "256Mi" - restartPolicy: Always + imagePullSecrets: + - name: regcred + volumeClaimTemplates: + - metadata: + name: bot-storage + spec: + accessModes: + - ReadWriteOnce # Ensures only one node can mount the volume at a time + resources: + requests: + storage: 1Gi # Defines the storage size +--- +apiVersion: v1 +kind: Service +metadata: + name: bot-headless + labels: + app: bot +spec: + clusterIP: None + selector: + app: bot diff --git a/k8s/citus-coordinator.yml b/k8s/citus-coordinator.yml deleted file mode 100644 index e8a24c5..0000000 --- a/k8s/citus-coordinator.yml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: citus-coordinator -spec: - replicas: 1 - selector: - matchLabels: - app: citus - role: coordinator - template: - metadata: - labels: - app: citus - role: coordinator - spec: - containers: - - name: coordinator - image: citusdata/citus:latest - env: - - name: POSTGRES_DB - value: "payphone" - - name: POSTGRES_USER - value: "payphone" - - name: POSTGRES_PASSWORD - value: "payphone" - volumeMounts: - - name: citus-coordinator-storage - mountPath: /var/lib/postgresql/data - ports: - - containerPort: 5432 - resources: - requests: - memory: "2Gi" - cpu: "1" - limits: - memory: "2Gi" - cpu: "2" - volumes: - - name: citus-coordinator-storage - persistentVolumeClaim: - claimName: citus-coordinator-pvc - ---- -apiVersion: v1 -kind: Service -metadata: - name: citus-coordinator -spec: - selector: - app: citus - role: coordinator - ports: - - protocol: TCP - port: 5432 - targetPort: 5432 - type: ClusterIP diff --git a/k8s/citus-pvc.yml b/k8s/citus-pvc.yml deleted file mode 100644 index 26f859c..0000000 --- a/k8s/citus-pvc.yml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: citus-coordinator-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi # Change this size as needed - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: citus-worker-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi # Change this size as needed diff --git a/k8s/citus-worker.yml b/k8s/citus.yml similarity index 60% rename from k8s/citus-worker.yml rename to k8s/citus.yml index 068973b..76d5d94 100644 --- a/k8s/citus-worker.yml +++ b/k8s/citus.yml @@ -1,22 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: citus-headless +spec: + selector: + app: citus + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 + clusterIP: None # Headless service for direct pod access + +--- apiVersion: apps/v1 kind: StatefulSet metadata: - name: citus-worker + name: citus spec: - replicas: 2 # Increase as needed for more workers + serviceName: citus-headless + replicas: 3 # Set the desired number of replicas selector: matchLabels: app: citus - role: worker - serviceName: citus-worker-headless template: metadata: labels: app: citus - role: worker spec: containers: - - name: worker + - name: citus image: citusdata/citus:latest env: - name: POSTGRES_DB @@ -27,37 +39,15 @@ spec: value: "payphone" ports: - containerPort: 5432 - resources: - requests: - memory: "2Gi" - cpu: "1" - limits: - memory: "2Gi" - cpu: "2" volumeMounts: - - name: citus-worker-storage + - name: citus-storage mountPath: /var/lib/postgresql/data volumeClaimTemplates: - metadata: - name: citus-worker-storage + name: citus-storage spec: accessModes: [ "ReadWriteOnce" ] storageClassName: "standard" # Update based on your storage class resources: requests: - storage: 5Gi # Size per worker - ---- -apiVersion: v1 -kind: Service -metadata: - name: citus-worker-headless -spec: - selector: - app: citus - role: worker - ports: - - protocol: TCP - port: 5432 - targetPort: 5432 - clusterIP: None # Defines the service as headless, allowing direct pod DNS entries + storage: 5Gi # Size for each pod's storage