From c0616cbcfeaacd5bed485744c7e513c1db5ff043 Mon Sep 17 00:00:00 2001 From: "Nyeki.py" <84998222+Nekidev@users.noreply.github.com> Date: Thu, 31 Oct 2024 04:57:58 -0300 Subject: [PATCH] feat(bot): Update stuff to migrate to k8s. --- .dockerignore | 2 + .env.example | 7 -- Dockerfile | 3 +- README.md | 110 +++++++++++++++++---- compose.yml | 36 ------- deploy.sh | 28 ------ k8s/bot.yml | 19 ++-- k8s/citus.yml | 14 +++ k8s/config.yml | 7 ++ k8s/secrets.yml | 7 ++ node-ips.txt | 2 - payphone/bot.py | 24 +---- payphone/core/config.py | 5 +- run.old.py | 211 +++++++++++++++++++++++++++++++++++++++ run.py | 213 ++++------------------------------------ run.sh | 9 -- 16 files changed, 367 insertions(+), 330 deletions(-) delete mode 100644 .env.example delete mode 100644 compose.yml delete mode 100644 deploy.sh create mode 100644 k8s/config.yml create mode 100644 k8s/secrets.yml delete mode 100644 node-ips.txt create mode 100644 run.old.py delete mode 100644 run.sh diff --git a/.dockerignore b/.dockerignore index 82f9275..75fa7bf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -160,3 +160,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +k8s/ \ No newline at end of file diff --git a/.env.example b/.env.example deleted file mode 100644 index ff6cfc6..0000000 --- a/.env.example +++ /dev/null @@ -1,7 +0,0 @@ -TOKEN= - -IS_DEBUG=false - -SUGGESTIONS_CHANNEL_ID= - -SERVER_INVITE= diff --git a/Dockerfile b/Dockerfile index df17e8c..d65d976 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ COPY . . RUN pip install uv RUN uv sync --no-dev -RUN uv add uvloop ENTRYPOINT [ "/bin/bash" ] + +CMD [ "uv", "run", "--no-dev", "python3", "-u", "run.py" ] diff --git a/README.md b/README.md index f2459ec..27d098d 100644 --- a/README.md +++ b/README.md @@ -98,39 +98,107 @@ The following steps show how to pull and install Payphone on a running cluster o ### Building and Publishing the Payphone Image -To be able to deploy Payphone to your k8s cluster, you first need to upload the image to a private -registry. This documentation uses Docker Hub as a registry +Payphone's container images are hosted here at `git.nyeki.dev`. To login, run the following +command: ```sh -# Only run this step if you need to build the source code and it is in your machine. Build and -# publish to the private registry. Use your own image name, for example `samantha/payphone:latest`. -docker login +docker login git.nyeki.dev +``` -docker build -t nekidev/payphone:latest . -docker publish nekidev/payphone:latest +A new browser tab will open and you'll be prompted for log in. Once you accept, you'll be logged in +to the container registry. -# In the following steps, if you're using microk8s in production, use `microk8s kubectl` instead of -# `kubectl`. You can add this to your ~/.bash_aliases to avoid having to prepend `microk8s` to all -# commands: -# alias kubectl='microk8s kubectl' +When you have made an update and want to update the production deployment's image, you need to +build and push the image to the container registry: -# This step requires you to create a personal access token for Docker Hub. To create it: -# 1. Go to https://app.docker.com/settings/personal-access-tokens/create -# 2. Set the access permission to "Read-only" -kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username= --docker-password= +```sh +docker build -t git.nyeki.dev/payphone/payphone:latest . +docker push git.nyeki.dev/payphone/payphone:latest +``` +### Starting Payphone in the Cluster -# This will pull the images for citus and redis and start their containers. -kubectl apply -f redis.yml -kubectl apply -f citus.yml +#### Setting Up Config Files -# After applying citus.yml, you'll need to connect the nodes. We'll always use citus-0 as the -# master instance. +First of all, you'll need to set up the environment variables that Payphone needs. For this, +you'll need: +1. A Discord bot token, base64 encoded. You can encode your token at + [www.base64encode.org](https://www.base64encode.org/). +2. A suggestions channel that the bot can write to (its ID). +3. A non-expiring invite to the official server. + +First of all, open `k8s/config.yml`. At the bottom of the file, you'll find a `data` object with +some environment variables definitions. Set `SUGGESTIONS_CHANNEL_ID` and `SERVER_INVITE` and save +the file. + +Then, after you've base64-encoded your token, open `k8s/secrets.yml` and in a similar `data` object +at the bottom of the file, set `TOKEN` to the base64-encoded Discord bot token and save the +changes. + +After the files are updated with their new values, run the following commands to apply these +changes: + +```sh +kubectl apply -f k8s/config.yml +kubectl apply -f k8s/secrets.yml +``` + +If you see a ConfigMap and a Secret being created, that means you've successfully applied the +changes. + +#### Setting Up Redis + +Now, you can initialize Redis. This step is pretty straightforward and you can forget about it +later. + +```sh +kubectl apply -f k8s/redis.yml +``` + +#### Setting Up Citus Database + +This step requires some updates in the `k8s/citus.yml` file. Open the file and make sure that the +default amount of replicas fits your needs. At least one is needed. + +```sh +kubectl apply -f k8s/citus.yml +``` + +Once all the nodes are up and running, you have to connect them. The node `citus-0` is always used +as the coordinator node. Run the following command to set the coordinator host: + +```sh +kubectl exec citus-0 -- psql -U payphone payphone -c "SELECT citus_set_coordinator_host('citus-0.citus-headless.default.svc.cluster.local', 5432);" +``` + +Once that command was run, you need to add the rest of the nodes to the cluster. To do that, run +the following command replacing `` with the index of the node you're adding, once per +node. + +```sh +kubectl exec citus-0 -- psql -U payphone payphone -c "SELECT * from citus_add_node('citus-.citus-headless.default.svc.cluster.local', 5432);" +``` + +For example, if you had 3 replicas in total, you'd run the command 2 times, once for node 1 and +once for node 2 (node 0 is the coordinator, so you don't have to add it). + +As an optional step, you can check that the nodes were successfully connected running the following +command, which will show all the nodes connected: + +```sh +kubectl exec citus-0 -- psql -U payphone payphone -c "SELECT * FROM citus_get_active_worker_nodes();" +``` + +#### Setting Up Payphone + +Finally, you need to start the bot's instance. + +```sh # Steps before running this command: # 1. Update the image name in `bot.yml` to your own image name (the one that you have in your # private docker hub registry) # 2. Set the total number of shards in both `spec.replicas` and # `spec.template.spec.containers.bot.env.TOTAL_SHARDS.value`. -kubectl apply -f bot.yml +kubectl apply -f k8s/bot.yml ``` diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 49b796d..0000000 --- a/compose.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: "payphone" - -services: - bot: - build: . - env_file: - - .env - environment: - IS_DEBUG: "false" - depends_on: - - database - - redis - command: run.sh - restart: always - - database: - image: postgres:16.3 - environment: - - POSTGRES_DB=payphone - - POSTGRES_USER=payphone - - POSTGRES_PASSWORD=payphone - volumes: - - database:/var/lib/postgresql/data - command: postgres -c 'max_connections=1000' - # ports: - # - 5432:5432 - restart: always - - redis: - image: redis:7.4.1-alpine - # ports: - # - 6379:6379 - restart: always - -volumes: - database: diff --git a/deploy.sh b/deploy.sh deleted file mode 100644 index 0f1bb37..0000000 --- a/deploy.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -# Check for updates. -git pull - -# Create the enviroment file. -if [ ! -f ".env" ]; then - # If the destination file doesn't exist, copy the source file to the destination - cp ".env.example" ".env" - - if [ -z "$EDITOR" ]; then - EDITOR="nano" - fi - - $EDITOR .env -else - echo ".env file already exists. Make sure to check for updates in the .env.example file." -fi - -# Build image and create the container. -docker compose down - -docker compose build - -docker compose up -d - -echo "Everything's set up and running!" -echo "You can stop the bot running 'docker compose down'." diff --git a/k8s/bot.yml b/k8s/bot.yml index 1b840be..3d2281a 100644 --- a/k8s/bot.yml +++ b/k8s/bot.yml @@ -6,7 +6,7 @@ metadata: app: bot spec: serviceName: "bot-headless" - replicas: 2 + replicas: 1 # Adjust this manually together with env.TOTAL_SHARDS down below. selector: matchLabels: app: bot @@ -17,7 +17,12 @@ spec: spec: containers: - name: bot - image: nekidev/payphone:latest + image: git.nyeki.dev/payphone/payphone:latest + envFrom: + - configMapRef: + name: bot-config + - secretsRef: + name: bot-secrets env: - name: IS_DEBUG value: "false" @@ -25,17 +30,15 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] - - name: TOTAL_SHARDS - value: "2" # Adjust this manually or with a ConfigMap + - name: SHARD_COUNT + value: "1" # Adjust this manually together with spec.replicas up at the top. resources: limits: - cpu: "500m" - memory: "512Mi" + cpu: "1000m" + memory: "1Gi" requests: cpu: "250m" memory: "256Mi" - imagePullSecrets: - - name: regcred volumeClaimTemplates: - metadata: name: bot-storage diff --git a/k8s/citus.yml b/k8s/citus.yml index 76d5d94..e7a4ea3 100644 --- a/k8s/citus.yml +++ b/k8s/citus.yml @@ -1,5 +1,19 @@ apiVersion: v1 kind: Service +metadata: + name: citus-loadbalancer +spec: + type: LoadBalancer + selector: + app: citus + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 + +--- +apiVersion: v1 +kind: Service metadata: name: citus-headless spec: diff --git a/k8s/config.yml b/k8s/config.yml new file mode 100644 index 0000000..494e90a --- /dev/null +++ b/k8s/config.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: bot-config +data: + SUGGESTIONS_CHANNEL_ID: "1287841493343801447" + SERVER_INVITE: "https://discord.gg/Cb3rCrTxmm" diff --git a/k8s/secrets.yml b/k8s/secrets.yml new file mode 100644 index 0000000..6fd2315 --- /dev/null +++ b/k8s/secrets.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: bot-secrets +type: Opaque +data: + TOKEN: TVRJMU56ZzVOemMxT1RRM09URTNNekl4TWcuR2J0d0dsLk9IVGJmN2JScVZHa0tTeXNPTzRGUXAzQnV0MmxISUF6dGpuZnhz diff --git a/node-ips.txt b/node-ips.txt deleted file mode 100644 index 5d99aa5..0000000 --- a/node-ips.txt +++ /dev/null @@ -1,2 +0,0 @@ -45.134.39.59 -2.56.246.33 diff --git a/payphone/bot.py b/payphone/bot.py index ed0f261..e1105e7 100644 --- a/payphone/bot.py +++ b/payphone/bot.py @@ -29,29 +29,11 @@ from payphone.core import ( class Payphone(commands.AutoShardedBot): - shutdown: managers.ValueProxy[bool] - """A value proxy that connects with all other shard processes. - - This value is shared between shard processes, and if any of them sets this proxy's value to - `True`, then all shard processes will be terminated by the main process (`run.py`). - - To check if the shutdown has been scheduled (but not necessarily is immediately about to - occur), use `scheduled_shutdown` instead. - """ - - scheduled_shutdown: managers.ValueProxy[bool] - """A value proxy that connects with all other shard processes. - - This value is shared between shard processes, and if any of them sets this proxy's value to - `True`, then the bot's shutdown has been started and all shards will be terminated soon. - - To check whether the bot's about to terminate (if the message was sent to the main process) use - `shutdown`. - """ - log = logging.logger - redis: aredis.Redis = aredis.Redis(host="localhost" if config.IS_DEBUG else "redis") + redis: aredis.Redis = aredis.Redis( + host="localhost" if config.IS_DEBUG else "redis.default.svc.cluster.local" + ) def __init__(self) -> None: intents = discord.Intents.default() diff --git a/payphone/core/config.py b/payphone/core/config.py index f4f08e9..2fa3279 100644 --- a/payphone/core/config.py +++ b/payphone/core/config.py @@ -7,6 +7,9 @@ load_dotenv() TOKEN = os.getenv("TOKEN") +SHARD_ID = int(os.getenv("SHARD_ID", "0")) +SHARD_COUNT = int(os.getenv("SHARD_COUNT", "1")) + IS_DEBUG = os.environ.get("IS_DEBUG", "false") == "true" TORTOISE_ORM = { @@ -14,7 +17,7 @@ TORTOISE_ORM = { "default": { "engine": "tortoise.backends.asyncpg", "credentials": { - "host": "localhost" if IS_DEBUG else "database", + "host": "localhost" if IS_DEBUG else "citus-loadbalancer.default.svc.cluster.local", "port": "5432", "user": "payphone", "password": "payphone", diff --git a/run.old.py b/run.old.py new file mode 100644 index 0000000..b4ed4aa --- /dev/null +++ b/run.old.py @@ -0,0 +1,211 @@ +from multiprocessing import managers + +from datetime import datetime, timezone + +from payphone.core import logging + + +log = logging.logger + + +def shard( + shard_id: int, + shard_count: int, + shutdown: managers.ValueProxy[bool], + scheduled_shutdown: managers.ValueProxy[bool], +) -> None: + """Starts a new shard. + + Args: + shard_id (int): The shard's ID. + shard_count (int): The total amount of shards. + shutdown (ValueProxy[bool]): The flag for the main process to shut down all shards. + scheduled_shutdown (ValueProxy[bool]): The flag for other shards to know the bot's gonna + shut down soon. + """ + + from payphone.db import setup + from payphone.bot import bot + from payphone.core import config, logging + + bot.shard_ids = [shard_id] + bot.shard_count = shard_count + bot.shutdown = shutdown + bot.scheduled_shutdown = scheduled_shutdown + + logging.setup(f"Shard {shard_id}") + + log.info("Running shard %s with default loop.", shard_id) + + bot.loop.set_debug(True) + + async def initialize(): + await setup.init() + await bot.redis.ping() + + bot.loop.run_until_complete(initialize()) + + bot.run(config.TOKEN) + + +def flush_process(): + import asyncio + + from procs.flush_messages import loop + + try: + import uvloop # pylint: disable=import-error + + uvloop.install() + log.info("Flushing messages with uvloop.") + + except ImportError: + log.info("Flushing messages with default loop.") + + finally: + asyncio.run(loop()) + + +class ShutdownException(Exception): + """Raised when a shard has triggered the shutdown of all shards.""" + + +if __name__ == "__main__": + import multiprocessing as mp + + import time + + from tortoise import run_async + + import httpx + + import redis.asyncio as aredis + + from payphone.db import setup, models + from payphone.core import config + + logging.setup("Payphone") + + redis = aredis.Redis(host="localhost" if config.IS_DEBUG else "redis") + + async def init(): + await setup.init() + + await models.Call.filter(ended_at=None).update( + ended_at=datetime.now(timezone.utc) + ) + await models.Channel.exclude(call_id=None).update(call_id=None) + + await redis.flushall() + + log.info("Initializing DB and running Redis cleanup...") + + run_async(init()) + + log.info("Cleanup done! Loading recommended shard amount...") + + response = httpx.get( + "https://discord.com/api/v10/gateway/bot", + headers={"Authorization": f"Bot {config.TOKEN}"}, + ) + + info = response.json() + + shard_count = info["shards"] + + manager = mp.Manager() + shutdown = manager.Value("b", False) + scheduled_shutdown = manager.Value("b", False) + + log.info("Starting %s shards.", shard_count) + + ps: list[mp.Process] = [ + mp.Process( + target=shard, + args=(i, shard_count, shutdown, scheduled_shutdown), + ) + for i in range(shard_count) + ] + flushing_p = mp.Process(target=flush_process) + + for i, p in enumerate(ps): + p.start() + + log.info("Shard %s's process started.", i) + log.info( + "Waiting 5 seconds start initialize next shard..." + if i != len(ps) - 1 + else "All shards were started." + ) + + if i != len(ps) - 1: + time.sleep(5) + + log.info("Starting messages flushing process...") + + flushing_p.start() + + log.info("Messages flushing process started.") + + try: + while True: + if shutdown.value: + raise ShutdownException() + + for i in range(shard_count): + if not ps[i].is_alive(): + log.warning("Shard %s's process exited. Restarting the shard...", i) + + ps[i] = mp.Process( + target=shard, + args=( + i, + shard_count, + shutdown, + scheduled_shutdown, + ), + name=f"payphone-shard-{i}", + ) + ps[i].start() + + log.info( + "Shard %s was restarted. Waiting 5 seconds to check next shard...", + i, + ) + time.sleep(5) + + if not flushing_p.is_alive(): + log.info("Messages flushing process was stopped. Restarting it...") + + flushing_p = mp.Process(target=flush_process, name="payphone-flushing") + flushing_p.start() + + log.info("Messages flushing process started.") + + time.sleep(5) + + except KeyboardInterrupt: + log.info("Ctrl-C was pressed.") + + except ShutdownException: + log.info("A shard shut down the bot.") + + finally: + log.info("Terminating %s shards...", shard_count) + + for i, p in enumerate(ps): + p.terminate() + log.info("Terminated shard %s.", i) + + log.info("Stopping messages flushing process...") + + flushing_p.terminate() + + log.info("Checking that all processes have finished...") + + for p in ps: + p.join() + + flushing_p.join() + + log.info("All processes have been terminated.") diff --git a/run.py b/run.py index b4ed4aa..f619690 100644 --- a/run.py +++ b/run.py @@ -1,211 +1,32 @@ -from multiprocessing import managers +import aerich -from datetime import datetime, timezone +from payphone import bot +from payphone.db import setup +from payphone.core import config, logging -from payphone.core import logging +bot = bot.bot -log = logging.logger +bot.shard_ids = [config.SHARD_ID] +bot.shard_count = config.SHARD_COUNT -def shard( - shard_id: int, - shard_count: int, - shutdown: managers.ValueProxy[bool], - scheduled_shutdown: managers.ValueProxy[bool], -) -> None: - """Starts a new shard. - Args: - shard_id (int): The shard's ID. - shard_count (int): The total amount of shards. - shutdown (ValueProxy[bool]): The flag for the main process to shut down all shards. - scheduled_shutdown (ValueProxy[bool]): The flag for other shards to know the bot's gonna - shut down soon. - """ +logging.setup("Payphone") - from payphone.db import setup - from payphone.bot import bot - from payphone.core import config, logging - bot.shard_ids = [shard_id] - bot.shard_count = shard_count - bot.shutdown = shutdown - bot.scheduled_shutdown = scheduled_shutdown +async def init() -> None: + """Initializes the DB.""" - logging.setup(f"Shard {shard_id}") + if config.SHARD_ID == 0: + command = aerich.Command(config.TORTOISE_ORM, "models", "./payphone/db/migrations") - log.info("Running shard %s with default loop.", shard_id) + await command.init_db(True) + await command.upgrade() - bot.loop.set_debug(True) + await setup.init() - async def initialize(): - await setup.init() - await bot.redis.ping() - bot.loop.run_until_complete(initialize()) +bot.loop.run_until_complete(init()) - bot.run(config.TOKEN) - - -def flush_process(): - import asyncio - - from procs.flush_messages import loop - - try: - import uvloop # pylint: disable=import-error - - uvloop.install() - log.info("Flushing messages with uvloop.") - - except ImportError: - log.info("Flushing messages with default loop.") - - finally: - asyncio.run(loop()) - - -class ShutdownException(Exception): - """Raised when a shard has triggered the shutdown of all shards.""" - - -if __name__ == "__main__": - import multiprocessing as mp - - import time - - from tortoise import run_async - - import httpx - - import redis.asyncio as aredis - - from payphone.db import setup, models - from payphone.core import config - - logging.setup("Payphone") - - redis = aredis.Redis(host="localhost" if config.IS_DEBUG else "redis") - - async def init(): - await setup.init() - - await models.Call.filter(ended_at=None).update( - ended_at=datetime.now(timezone.utc) - ) - await models.Channel.exclude(call_id=None).update(call_id=None) - - await redis.flushall() - - log.info("Initializing DB and running Redis cleanup...") - - run_async(init()) - - log.info("Cleanup done! Loading recommended shard amount...") - - response = httpx.get( - "https://discord.com/api/v10/gateway/bot", - headers={"Authorization": f"Bot {config.TOKEN}"}, - ) - - info = response.json() - - shard_count = info["shards"] - - manager = mp.Manager() - shutdown = manager.Value("b", False) - scheduled_shutdown = manager.Value("b", False) - - log.info("Starting %s shards.", shard_count) - - ps: list[mp.Process] = [ - mp.Process( - target=shard, - args=(i, shard_count, shutdown, scheduled_shutdown), - ) - for i in range(shard_count) - ] - flushing_p = mp.Process(target=flush_process) - - for i, p in enumerate(ps): - p.start() - - log.info("Shard %s's process started.", i) - log.info( - "Waiting 5 seconds start initialize next shard..." - if i != len(ps) - 1 - else "All shards were started." - ) - - if i != len(ps) - 1: - time.sleep(5) - - log.info("Starting messages flushing process...") - - flushing_p.start() - - log.info("Messages flushing process started.") - - try: - while True: - if shutdown.value: - raise ShutdownException() - - for i in range(shard_count): - if not ps[i].is_alive(): - log.warning("Shard %s's process exited. Restarting the shard...", i) - - ps[i] = mp.Process( - target=shard, - args=( - i, - shard_count, - shutdown, - scheduled_shutdown, - ), - name=f"payphone-shard-{i}", - ) - ps[i].start() - - log.info( - "Shard %s was restarted. Waiting 5 seconds to check next shard...", - i, - ) - time.sleep(5) - - if not flushing_p.is_alive(): - log.info("Messages flushing process was stopped. Restarting it...") - - flushing_p = mp.Process(target=flush_process, name="payphone-flushing") - flushing_p.start() - - log.info("Messages flushing process started.") - - time.sleep(5) - - except KeyboardInterrupt: - log.info("Ctrl-C was pressed.") - - except ShutdownException: - log.info("A shard shut down the bot.") - - finally: - log.info("Terminating %s shards...", shard_count) - - for i, p in enumerate(ps): - p.terminate() - log.info("Terminated shard %s.", i) - - log.info("Stopping messages flushing process...") - - flushing_p.terminate() - - log.info("Checking that all processes have finished...") - - for p in ps: - p.join() - - flushing_p.join() - - log.info("All processes have been terminated.") +bot.run(config.TOKEN) diff --git a/run.sh b/run.sh deleted file mode 100644 index 58f2e67..0000000 --- a/run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Retry migrations until success. -until uv run --no-dev aerich upgrade -do - echo "Retrying migrations..." -done - -uv run --no-dev python3 -u run.py