feat(bot): Update stuff to migrate to k8s.

.dockerignore

@@ -160,3 +160,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+k8s/

.env.example

@@ -1,7 +0,0 @@
-TOKEN=
-
-IS_DEBUG=false
-
-SUGGESTIONS_CHANNEL_ID=
-
-SERVER_INVITE=

Dockerfile

@@ -5,6 +5,7 @@ COPY . .
 
 RUN pip install uv
 RUN uv sync --no-dev
-RUN uv add uvloop
 
 ENTRYPOINT [ "/bin/bash" ]
+
+CMD [ "uv", "run", "--no-dev", "python3", "-u", "run.py" ]

README.md

@@ -98,39 +98,107 @@ The following steps show how to pull and install Payphone on a running cluster o
 
 ### Building and Publishing the Payphone Image
 
-To be able to deploy Payphone to your k8s cluster, you first need to upload the image to a private
+Payphone's container images are hosted here at `git.nyeki.dev`. To login, run the following
-registry. This documentation uses Docker Hub as a registry
+command:
 
 ```sh
-# Only run this step if you need to build the source code and it is in your machine. Build and
+docker login git.nyeki.dev
-# publish to the private registry. Use your own image name, for example `samantha/payphone:latest`.
+```
-docker login
 
-docker build -t nekidev/payphone:latest .
+A new browser tab will open and you'll be prompted for log in. Once you accept, you'll be logged in
-docker publish nekidev/payphone:latest
+to the container registry.
 
-# In the following steps, if you're using microk8s in production, use `microk8s kubectl` instead of
+When you have made an update and want to update the production deployment's image, you need to
-# `kubectl`. You can add this to your ~/.bash_aliases to avoid having to prepend `microk8s` to all
+build and push the image to the container registry:
-# commands:
-# alias kubectl='microk8s kubectl'
 
-# This step requires you to create a personal access token for Docker Hub. To create it:
+```sh
-# 1. Go to https://app.docker.com/settings/personal-access-tokens/create
+docker build -t git.nyeki.dev/payphone/payphone:latest .
-# 2. Set the access permission to "Read-only"
+docker push git.nyeki.dev/payphone/payphone:latest
-kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<your-username> --docker-password=<your-personal-access-token>
+```
 
+### Starting Payphone in the Cluster
 
-# This will pull the images for citus and redis and start their containers.
+#### Setting Up Config Files
-kubectl apply -f redis.yml
-kubectl apply -f citus.yml
 
-# After applying citus.yml, you'll need to connect the nodes. We'll always use citus-0 as the
+First of all, you'll need to set up the environment variables that Payphone needs. For this,
-# master instance.
+you'll need:
 
+1. A Discord bot token, base64 encoded. You can encode your token at
+   [www.base64encode.org](https://www.base64encode.org/).
+2. A suggestions channel that the bot can write to (its ID).
+3. A non-expiring invite to the official server.
+
+First of all, open `k8s/config.yml`. At the bottom of the file, you'll find a `data` object with
+some environment variables definitions. Set `SUGGESTIONS_CHANNEL_ID` and `SERVER_INVITE` and save
+the file.
+
+Then, after you've base64-encoded your token, open `k8s/secrets.yml` and in a similar `data` object
+at the bottom of the file, set `TOKEN` to the base64-encoded Discord bot token and save the
+changes.
+
+After the files are updated with their new values, run the following commands to apply these
+changes:
+
+```sh
+kubectl apply -f k8s/config.yml
+kubectl apply -f k8s/secrets.yml
+```
+
+If you see a ConfigMap and a Secret being created, that means you've successfully applied the
+changes.
+
+#### Setting Up Redis
+
+Now, you can initialize Redis. This step is pretty straightforward and you can forget about it
+later.
+
+```sh
+kubectl apply -f k8s/redis.yml
+```
+
+#### Setting Up Citus Database
+
+This step requires some updates in the `k8s/citus.yml` file. Open the file and make sure that the
+default amount of replicas fits your needs. At least one is needed.
+
+```sh
+kubectl apply -f k8s/citus.yml
+```
+
+Once all the nodes are up and running, you have to connect them. The node `citus-0` is always used
+as the coordinator node. Run the following command to set the coordinator host:
+
+```sh
+kubectl exec citus-0 -- psql -U payphone payphone -c "SELECT citus_set_coordinator_host('citus-0.citus-headless.default.svc.cluster.local', 5432);"
+```
+
+Once that command was run, you need to add the rest of the nodes to the cluster. To do that, run
+the following command replacing `<node-index>` with the index of the node you're adding, once per
+node.
+
+```sh
+kubectl exec citus-0 -- psql -U payphone payphone -c "SELECT * from citus_add_node('citus-<node-index>.citus-headless.default.svc.cluster.local', 5432);"
+```
+
+For example, if you had 3 replicas in total, you'd run the command 2 times, once for node 1 and
+once for node 2 (node 0 is the coordinator, so you don't have to add it).
+
+As an optional step, you can check that the nodes were successfully connected running the following
+command, which will show all the nodes connected:
+
+```sh
+kubectl exec citus-0 -- psql -U payphone payphone -c "SELECT * FROM citus_get_active_worker_nodes();"
+```
+
+#### Setting Up Payphone
+
+Finally, you need to start the bot's instance.
+
+```sh
 # Steps before running this command:
 # 1. Update the image name in `bot.yml` to your own image name (the one that you have in your
 #      private docker hub registry)
 # 2. Set the total number of shards in both `spec.replicas` and
 #      `spec.template.spec.containers.bot.env.TOTAL_SHARDS.value`.
-kubectl apply -f bot.yml
+kubectl apply -f k8s/bot.yml
 ```

compose.yml

@@ -1,36 +0,0 @@
-name: "payphone"
-
-services:
-  bot:
-    build: .
-    env_file:
-      - .env
-    environment:
-      IS_DEBUG: "false"
-    depends_on:
-      - database
-      - redis
-    command: run.sh
-    restart: always
-
-  database:
-    image: postgres:16.3
-    environment:
-      - POSTGRES_DB=payphone
-      - POSTGRES_USER=payphone
-      - POSTGRES_PASSWORD=payphone
-    volumes:
-      - database:/var/lib/postgresql/data
-    command: postgres -c 'max_connections=1000'
-    # ports:
-    #   - 5432:5432
-    restart: always
-
-  redis:
-    image: redis:7.4.1-alpine
-    # ports:
-    #   - 6379:6379
-    restart: always
-
-volumes:
-  database:

deploy.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-# Check for updates.
-git pull
-
-# Create the enviroment file.
-if [ ! -f ".env" ]; then
-    # If the destination file doesn't exist, copy the source file to the destination
-    cp ".env.example" ".env"
-
-    if [ -z "$EDITOR" ]; then
-        EDITOR="nano"
-    fi
-
-    $EDITOR .env
-else
-    echo ".env file already exists. Make sure to check for updates in the .env.example file."
-fi
-
-# Build image and create the container.
-docker compose down
-
-docker compose build
-
-docker compose up -d
-
-echo "Everything's set up and running!"
-echo "You can stop the bot running 'docker compose down'."

k8s/bot.yml

@@ -6,7 +6,7 @@ metadata:
     app: bot
 spec:
   serviceName: "bot-headless"
-  replicas: 2
+  replicas: 1  # Adjust this manually together with env.TOTAL_SHARDS down below.
   selector:
     matchLabels:
       app: bot
@@ -17,7 +17,12 @@ spec:
     spec:
       containers:
         - name: bot
-          image: nekidev/payphone:latest
+          image: git.nyeki.dev/payphone/payphone:latest
+          envFrom:
+            - configMapRef:
+                name: bot-config
+            - secretsRef:
+                name: bot-secrets
           env:
             - name: IS_DEBUG
               value: "false"
@@ -25,17 +30,15 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.labels['apps.kubernetes.io/pod-index']
-            - name: TOTAL_SHARDS
+            - name: SHARD_COUNT
-              value: "2"  # Adjust this manually or with a ConfigMap
+              value: "1"  # Adjust this manually together with spec.replicas up at the top.
           resources:
             limits:
-              cpu: "500m"
+              cpu: "1000m"
-              memory: "512Mi"
+              memory: "1Gi"
             requests:
               cpu: "250m"
               memory: "256Mi"
-      imagePullSecrets:
-        - name: regcred
   volumeClaimTemplates:
     - metadata:
         name: bot-storage

k8s/citus.yml

@@ -1,5 +1,19 @@
 apiVersion: v1
 kind: Service
+metadata:
+  name: citus-loadbalancer
+spec:
+  type: LoadBalancer
+  selector:
+    app: citus
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+
+---
+apiVersion: v1
+kind: Service
 metadata:
   name: citus-headless
 spec:

k8s/config.yml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bot-config
+data:
+  SUGGESTIONS_CHANNEL_ID: "1287841493343801447"
+  SERVER_INVITE: "https://discord.gg/Cb3rCrTxmm"

k8s/secrets.yml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: bot-secrets
+type: Opaque
+data:
+  TOKEN: TVRJMU56ZzVOemMxT1RRM09URTNNekl4TWcuR2J0d0dsLk9IVGJmN2JScVZHa0tTeXNPTzRGUXAzQnV0MmxISUF6dGpuZnhz

node-ips.txt

@@ -1,2 +0,0 @@
-45.134.39.59
-2.56.246.33

payphone/bot.py

@@ -29,29 +29,11 @@ from payphone.core import (
 
 
 class Payphone(commands.AutoShardedBot):
-    shutdown: managers.ValueProxy[bool]
-    """A value proxy that connects with all other shard processes.
-    
-    This value is shared between shard processes, and if any of them sets this proxy's value to
-    `True`, then all shard processes will be terminated by the main process (`run.py`).
-
-    To check if the shutdown has been scheduled (but not necessarily is immediately about to
-    occur), use `scheduled_shutdown` instead.
-    """
-
-    scheduled_shutdown: managers.ValueProxy[bool]
-    """A value proxy that connects with all other shard processes.
-
-    This value is shared between shard processes, and if any of them sets this proxy's value to
-    `True`, then the bot's shutdown has been started and all shards will be terminated soon.
-
-    To check whether the bot's about to terminate (if the message was sent to the main process) use
-    `shutdown`.
-    """
-
     log = logging.logger
 
-    redis: aredis.Redis = aredis.Redis(host="localhost" if config.IS_DEBUG else "redis")
+    redis: aredis.Redis = aredis.Redis(
+        host="localhost" if config.IS_DEBUG else "redis.default.svc.cluster.local"
+    )
 
     def __init__(self) -> None:
         intents = discord.Intents.default()

payphone/core/config.py

@@ -7,6 +7,9 @@ load_dotenv()
 
 TOKEN = os.getenv("TOKEN")
 
+SHARD_ID = int(os.getenv("SHARD_ID", "0"))
+SHARD_COUNT = int(os.getenv("SHARD_COUNT", "1"))
+
 IS_DEBUG = os.environ.get("IS_DEBUG", "false") == "true"
 
 TORTOISE_ORM = {
@@ -14,7 +17,7 @@ TORTOISE_ORM = {
         "default": {
             "engine": "tortoise.backends.asyncpg",
             "credentials": {
-                "host": "localhost" if IS_DEBUG else "database",
+                "host": "localhost" if IS_DEBUG else "citus-loadbalancer.default.svc.cluster.local",
                 "port": "5432",
                 "user": "payphone",
                 "password": "payphone",

run.old.py

@@ -0,0 +1,211 @@
+from multiprocessing import managers
+
+from datetime import datetime, timezone
+
+from payphone.core import logging
+
+
+log = logging.logger
+
+
+def shard(
+    shard_id: int,
+    shard_count: int,
+    shutdown: managers.ValueProxy[bool],
+    scheduled_shutdown: managers.ValueProxy[bool],
+) -> None:
+    """Starts a new shard.
+
+    Args:
+        shard_id (int): The shard's ID.
+        shard_count (int): The total amount of shards.
+        shutdown (ValueProxy[bool]): The flag for the main process to shut down all shards.
+        scheduled_shutdown (ValueProxy[bool]): The flag for other shards to know the bot's gonna
+          shut down soon.
+    """
+
+    from payphone.db import setup
+    from payphone.bot import bot
+    from payphone.core import config, logging
+
+    bot.shard_ids = [shard_id]
+    bot.shard_count = shard_count
+    bot.shutdown = shutdown
+    bot.scheduled_shutdown = scheduled_shutdown
+
+    logging.setup(f"Shard {shard_id}")
+
+    log.info("Running shard %s with default loop.", shard_id)
+
+    bot.loop.set_debug(True)
+
+    async def initialize():
+        await setup.init()
+        await bot.redis.ping()
+
+    bot.loop.run_until_complete(initialize())
+
+    bot.run(config.TOKEN)
+
+
+def flush_process():
+    import asyncio
+
+    from procs.flush_messages import loop
+
+    try:
+        import uvloop  # pylint: disable=import-error
+
+        uvloop.install()
+        log.info("Flushing messages with uvloop.")
+
+    except ImportError:
+        log.info("Flushing messages with default loop.")
+
+    finally:
+        asyncio.run(loop())
+
+
+class ShutdownException(Exception):
+    """Raised when a shard has triggered the shutdown of all shards."""
+
+
+if __name__ == "__main__":
+    import multiprocessing as mp
+
+    import time
+
+    from tortoise import run_async
+
+    import httpx
+
+    import redis.asyncio as aredis
+
+    from payphone.db import setup, models
+    from payphone.core import config
+
+    logging.setup("Payphone")
+
+    redis = aredis.Redis(host="localhost" if config.IS_DEBUG else "redis")
+
+    async def init():
+        await setup.init()
+
+        await models.Call.filter(ended_at=None).update(
+            ended_at=datetime.now(timezone.utc)
+        )
+        await models.Channel.exclude(call_id=None).update(call_id=None)
+
+        await redis.flushall()
+
+    log.info("Initializing DB and running Redis cleanup...")
+
+    run_async(init())
+
+    log.info("Cleanup done! Loading recommended shard amount...")
+
+    response = httpx.get(
+        "https://discord.com/api/v10/gateway/bot",
+        headers={"Authorization": f"Bot {config.TOKEN}"},
+    )
+
+    info = response.json()
+
+    shard_count = info["shards"]
+
+    manager = mp.Manager()
+    shutdown = manager.Value("b", False)
+    scheduled_shutdown = manager.Value("b", False)
+
+    log.info("Starting %s shards.", shard_count)
+
+    ps: list[mp.Process] = [
+        mp.Process(
+            target=shard,
+            args=(i, shard_count, shutdown, scheduled_shutdown),
+        )
+        for i in range(shard_count)
+    ]
+    flushing_p = mp.Process(target=flush_process)
+
+    for i, p in enumerate(ps):
+        p.start()
+
+        log.info("Shard %s's process started.", i)
+        log.info(
+            "Waiting 5 seconds start initialize next shard..."
+            if i != len(ps) - 1
+            else "All shards were started."
+        )
+
+        if i != len(ps) - 1:
+            time.sleep(5)
+
+    log.info("Starting messages flushing process...")
+
+    flushing_p.start()
+
+    log.info("Messages flushing process started.")
+
+    try:
+        while True:
+            if shutdown.value:
+                raise ShutdownException()
+
+            for i in range(shard_count):
+                if not ps[i].is_alive():
+                    log.warning("Shard %s's process exited. Restarting the shard...", i)
+
+                    ps[i] = mp.Process(
+                        target=shard,
+                        args=(
+                            i,
+                            shard_count,
+                            shutdown,
+                            scheduled_shutdown,
+                        ),
+                        name=f"payphone-shard-{i}",
+                    )
+                    ps[i].start()
+
+                    log.info(
+                        "Shard %s was restarted. Waiting 5 seconds to check next shard...",
+                        i,
+                    )
+                    time.sleep(5)
+
+            if not flushing_p.is_alive():
+                log.info("Messages flushing process was stopped. Restarting it...")
+
+                flushing_p = mp.Process(target=flush_process, name="payphone-flushing")
+                flushing_p.start()
+
+                log.info("Messages flushing process started.")
+
+            time.sleep(5)
+
+    except KeyboardInterrupt:
+        log.info("Ctrl-C was pressed.")
+
+    except ShutdownException:
+        log.info("A shard shut down the bot.")
+
+    finally:
+        log.info("Terminating %s shards...", shard_count)
+
+        for i, p in enumerate(ps):
+            p.terminate()
+            log.info("Terminated shard %s.", i)
+
+        log.info("Stopping messages flushing process...")
+
+        flushing_p.terminate()
+
+        log.info("Checking that all processes have finished...")
+
+        for p in ps:
+            p.join()
+
+        flushing_p.join()
+
+        log.info("All processes have been terminated.")

run.py

@@ -1,211 +1,32 @@
-from multiprocessing import managers
+import aerich
 
-from datetime import datetime, timezone
+from payphone import bot
-
+from payphone.db import setup
-from payphone.core import logging
+from payphone.core import config, logging
 
 
-log = logging.logger
+bot = bot.bot
 
 
-def shard(
+bot.shard_ids = [config.SHARD_ID]
-    shard_id: int,
+bot.shard_count = config.SHARD_COUNT
-    shard_count: int,
-    shutdown: managers.ValueProxy[bool],
-    scheduled_shutdown: managers.ValueProxy[bool],
-) -> None:
-    """Starts a new shard.
-
-    Args:
-        shard_id (int): The shard's ID.
-        shard_count (int): The total amount of shards.
-        shutdown (ValueProxy[bool]): The flag for the main process to shut down all shards.
-        scheduled_shutdown (ValueProxy[bool]): The flag for other shards to know the bot's gonna
-          shut down soon.
-    """
-
-    from payphone.db import setup
-    from payphone.bot import bot
-    from payphone.core import config, logging
-
-    bot.shard_ids = [shard_id]
-    bot.shard_count = shard_count
-    bot.shutdown = shutdown
-    bot.scheduled_shutdown = scheduled_shutdown
-
-    logging.setup(f"Shard {shard_id}")
-
-    log.info("Running shard %s with default loop.", shard_id)
-
-    bot.loop.set_debug(True)
-
-    async def initialize():
-        await setup.init()
-        await bot.redis.ping()
-
-    bot.loop.run_until_complete(initialize())
-
-    bot.run(config.TOKEN)
 
 
-def flush_process():
+logging.setup("Payphone")
-    import asyncio
-
-    from procs.flush_messages import loop
-
-    try:
-        import uvloop  # pylint: disable=import-error
-
-        uvloop.install()
-        log.info("Flushing messages with uvloop.")
-
-    except ImportError:
-        log.info("Flushing messages with default loop.")
-
-    finally:
-        asyncio.run(loop())
 
 
-class ShutdownException(Exception):
+async def init() -> None:
-    """Raised when a shard has triggered the shutdown of all shards."""
+    """Initializes the DB."""
 
+    if config.SHARD_ID == 0:
+        command = aerich.Command(config.TORTOISE_ORM, "models", "./payphone/db/migrations")
 
-if __name__ == "__main__":
+        await command.init_db(True)
-    import multiprocessing as mp
+        await command.upgrade()
 
-    import time
-
-    from tortoise import run_async
-
-    import httpx
-
-    import redis.asyncio as aredis
-
-    from payphone.db import setup, models
-    from payphone.core import config
-
-    logging.setup("Payphone")
-
-    redis = aredis.Redis(host="localhost" if config.IS_DEBUG else "redis")
-
-    async def init():
     await setup.init()
 
-        await models.Call.filter(ended_at=None).update(
-            ended_at=datetime.now(timezone.utc)
-        )
-        await models.Channel.exclude(call_id=None).update(call_id=None)
 
-        await redis.flushall()
+bot.loop.run_until_complete(init())
 
-    log.info("Initializing DB and running Redis cleanup...")
+bot.run(config.TOKEN)
-
-    run_async(init())
-
-    log.info("Cleanup done! Loading recommended shard amount...")
-
-    response = httpx.get(
-        "https://discord.com/api/v10/gateway/bot",
-        headers={"Authorization": f"Bot {config.TOKEN}"},
-    )
-
-    info = response.json()
-
-    shard_count = info["shards"]
-
-    manager = mp.Manager()
-    shutdown = manager.Value("b", False)
-    scheduled_shutdown = manager.Value("b", False)
-
-    log.info("Starting %s shards.", shard_count)
-
-    ps: list[mp.Process] = [
-        mp.Process(
-            target=shard,
-            args=(i, shard_count, shutdown, scheduled_shutdown),
-        )
-        for i in range(shard_count)
-    ]
-    flushing_p = mp.Process(target=flush_process)
-
-    for i, p in enumerate(ps):
-        p.start()
-
-        log.info("Shard %s's process started.", i)
-        log.info(
-            "Waiting 5 seconds start initialize next shard..."
-            if i != len(ps) - 1
-            else "All shards were started."
-        )
-
-        if i != len(ps) - 1:
-            time.sleep(5)
-
-    log.info("Starting messages flushing process...")
-
-    flushing_p.start()
-
-    log.info("Messages flushing process started.")
-
-    try:
-        while True:
-            if shutdown.value:
-                raise ShutdownException()
-
-            for i in range(shard_count):
-                if not ps[i].is_alive():
-                    log.warning("Shard %s's process exited. Restarting the shard...", i)
-
-                    ps[i] = mp.Process(
-                        target=shard,
-                        args=(
-                            i,
-                            shard_count,
-                            shutdown,
-                            scheduled_shutdown,
-                        ),
-                        name=f"payphone-shard-{i}",
-                    )
-                    ps[i].start()
-
-                    log.info(
-                        "Shard %s was restarted. Waiting 5 seconds to check next shard...",
-                        i,
-                    )
-                    time.sleep(5)
-
-            if not flushing_p.is_alive():
-                log.info("Messages flushing process was stopped. Restarting it...")
-
-                flushing_p = mp.Process(target=flush_process, name="payphone-flushing")
-                flushing_p.start()
-
-                log.info("Messages flushing process started.")
-
-            time.sleep(5)
-
-    except KeyboardInterrupt:
-        log.info("Ctrl-C was pressed.")
-
-    except ShutdownException:
-        log.info("A shard shut down the bot.")
-
-    finally:
-        log.info("Terminating %s shards...", shard_count)
-
-        for i, p in enumerate(ps):
-            p.terminate()
-            log.info("Terminated shard %s.", i)
-
-        log.info("Stopping messages flushing process...")
-
-        flushing_p.terminate()
-
-        log.info("Checking that all processes have finished...")
-
-        for p in ps:
-            p.join()
-
-        flushing_p.join()
-
-        log.info("All processes have been terminated.")

run.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# Retry migrations until success.
-until uv run --no-dev aerich upgrade
-do
-    echo "Retrying migrations..."
-done
-
-uv run --no-dev python3 -u run.py