From b2fb2f93802a86222dabbac5512969ca398765e0 Mon Sep 17 00:00:00 2001 From: Michail Kostochka Date: Mon, 16 Dec 2024 19:13:27 +0300 Subject: [PATCH] Various fixes - cleaned up Dockefiles - added healthchecks to docker-compose files - moved celery & celery-beat to one container - cleaned up nginx config --- .gitignore | 1 + backend/Dockerfile | 19 +----- backend/scripts/entrypoint.sh | 32 ---------- backend/scripts/gunicorn.sh | 0 backend/scripts/start.sh | 0 backend/scripts/start_celery.sh | 8 +-- batcher/Dockerfile | 6 +- batcher/app/src/db/pg/pg.py | 9 +-- batcher/app/src/db/rmq.py | 8 +-- bot/Dockerfile | 20 +----- bot/handlers/instruction.py | 2 +- bot/scripts/gunicorn.sh | 0 bot/scripts/start.sh | 0 docker-compose-prod.yml | 106 +++++++++++++------------------- docker-compose.yml | 77 +++++++++++------------ nginx/Dockerfile | 4 ++ nginx/nginx.conf | 42 +------------ 17 files changed, 99 insertions(+), 235 deletions(-) delete mode 100644 backend/scripts/entrypoint.sh mode change 100644 => 100755 backend/scripts/gunicorn.sh mode change 100644 => 100755 backend/scripts/start.sh mode change 100644 => 100755 backend/scripts/start_celery.sh mode change 100644 => 100755 bot/scripts/gunicorn.sh mode change 100644 => 100755 bot/scripts/start.sh diff --git a/.gitignore b/.gitignore index 9166dcc..bff3029 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ celerybeat-schedule backend/static backend/media bot/logs.log +bot/logfile.log diff --git a/backend/Dockerfile b/backend/Dockerfile index 68f8685..537ebb3 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,22 +1,7 @@ FROM python:3.11 -# python envs -ENV PYTHONFAULTHANDLER=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONHASHSEED=random \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 +COPY . /app -# python dependencies -COPY ./requirements.txt / -RUN pip install -r ./requirements.txt - -COPY ./scripts/entrypoint.sh ./scripts/start.sh ./scripts/gunicorn.sh ./scripts/start_celery.sh / -# upload scripts - -# Fix windows docker bug, convert CRLF to LF -RUN sed -i 's/\r$//g' /start.sh && chmod +x /start.sh && sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh &&\ - sed -i 's/\r$//g' /gunicorn.sh && chmod +x /gunicorn.sh && sed -i 's/\r$//g' /start_celery.sh && chmod +x /start_celery.sh +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt WORKDIR /app diff --git a/backend/scripts/entrypoint.sh b/backend/scripts/entrypoint.sh deleted file mode 100644 index 117bb25..0000000 --- a/backend/scripts/entrypoint.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail -cmd="$@" - -function postgres_ready(){ -python << END -import sys -import os -import psycopg2 - -try: - dbname = os.getenv('POSTGRES_DB') - user = os.getenv('POSTGRES_USER') - password = os.getenv('POSTGRES_PASSWORD') - host = os.getenv('DB_HOST', 'postgres') - port = os.getenv('POSTGRES_PORT', '5432') - conn = psycopg2.connect(dbname=dbname, user=user, password=password, host=host, port=port) -except psycopg2.OperationalError: - sys.exit(-1) -sys.exit(0) -END -} - -until postgres_ready; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 -done - ->&2 echo "Postgres is up - continuing..." -exec $cmd diff --git a/backend/scripts/gunicorn.sh b/backend/scripts/gunicorn.sh old mode 100644 new mode 100755 diff --git a/backend/scripts/start.sh b/backend/scripts/start.sh old mode 100644 new mode 100755 diff --git a/backend/scripts/start_celery.sh b/backend/scripts/start_celery.sh old mode 100644 new mode 100755 index e5c9660..d94e00c --- a/backend/scripts/start_celery.sh +++ b/backend/scripts/start_celery.sh @@ -1,7 +1,7 @@ -#!/usr/bin/env bash +#!/bin/sh -for i in {0..$CELERY_WORKER_COUNT} -do - celery -A clicker worker -l info --concurrency=10 -n worker$i@%h +for i in $(seq 1 "${CELERY_WORKER_COUNT}"); do + celery -A clicker worker -l info --concurrency=10 -n "worker${i}@$(%h)" done +celery -A clicker beat -l info diff --git a/batcher/Dockerfile b/batcher/Dockerfile index 7c5b6e8..82e8ea4 100644 --- a/batcher/Dockerfile +++ b/batcher/Dockerfile @@ -2,12 +2,10 @@ FROM python:3.12 WORKDIR /batcher -COPY ./requirements.txt /batcher/requirements.txt +COPY ./ /batcher RUN pip install --no-cache-dir --upgrade -r /batcher/requirements.txt -COPY ./app /batcher/app - ENV PYTHONPATH="${PYTHONPATH}:/batcher/app" -CMD uvicorn app.main:app --host 0.0.0.0 --port "${HTTP_PORT}" \ No newline at end of file +CMD uvicorn app.main:app --host 0.0.0.0 --port "${HTTP_PORT}" diff --git a/batcher/app/src/db/pg/pg.py b/batcher/app/src/db/pg/pg.py index ff0cc46..df38a40 100644 --- a/batcher/app/src/db/pg/pg.py +++ b/batcher/app/src/db/pg/pg.py @@ -15,14 +15,7 @@ MIGRATIONS_DIR = Path(__file__).parent.resolve() / "migrations" logger = logging.getLogger("uvicorn") async def connect_pg() -> asyncpg.Pool: - while True: - try: - logger.info(DB_URL) - pg_conn = await asyncpg.create_pool(DB_URL) - return pg_conn - except OSError: - logger.info("Postgres is unavailable - sleeping") - await asyncio.sleep(2) + return await asyncpg.create_pool(DB_URL) async def get_pg(request: Request) -> asyncpg.Connection: diff --git a/batcher/app/src/db/rmq.py b/batcher/app/src/db/rmq.py index c89a317..3287a67 100644 --- a/batcher/app/src/db/rmq.py +++ b/batcher/app/src/db/rmq.py @@ -11,13 +11,7 @@ fqdn = f'amqp://{RMQ_USER}:{str(RMQ_PASSWORD)}@{RMQ_HOST}:{RMQ_PORT}/' logger = logging.getLogger("uvicorn") async def get_connection() -> AbstractRobustConnection: - while True: - try: - conn = await aio_pika.connect_robust(fqdn) - return conn - except ConnectionError: - logger.info("RabbitMQ is unavailable - sleeping") - await asyncio.sleep(2) + return await aio_pika.connect_robust(fqdn) async def get_channel(conn_pool: AbstractRobustConnection) -> aio_pika.Channel: diff --git a/bot/Dockerfile b/bot/Dockerfile index 3d518a2..537ebb3 100644 --- a/bot/Dockerfile +++ b/bot/Dockerfile @@ -1,21 +1,7 @@ -FROM python:3.10.7 +FROM python:3.11 -# python envs -ENV PYTHONFAULTHANDLER=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONHASHSEED=random \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 - -# python dependencies -COPY ./requirements.txt / -RUN pip install -r /requirements.txt - -COPY ./scripts/start.sh ./scripts/gunicorn.sh / - -RUN chmod +x /start.sh -RUN chmod +x /gunicorn.sh +COPY . /app +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt WORKDIR /app diff --git a/bot/handlers/instruction.py b/bot/handlers/instruction.py index 74cfad5..6840de0 100644 --- a/bot/handlers/instruction.py +++ b/bot/handlers/instruction.py @@ -8,7 +8,7 @@ from aiogram.dispatcher.filters.state import State, StatesGroup from create_bot import bot, important_message, event_number from aiogram.types import ReplyKeyboardMarkup, KeyboardButton, ReplyKeyboardRemove, InlineKeyboardMarkup, InlineKeyboardButton from aiogram.utils.exceptions import MessageToDeleteNotFound -from memcached_def import add_rec, get_rec +from dbm_defs import add_rec, get_rec from loguru import logger diff --git a/bot/scripts/gunicorn.sh b/bot/scripts/gunicorn.sh old mode 100644 new mode 100755 diff --git a/bot/scripts/start.sh b/bot/scripts/start.sh old mode 100644 new mode 100755 diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 5316a20..64f57ac 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -2,20 +2,23 @@ volumes: db_data: {} batcher_db_data: {} redis_data: {} + backend_media: {} + certbot_www: {} + certbot_conf: {} services: backend: build: context: ./backend volumes: - - ./backend:/app - command: /gunicorn.sh - entrypoint: /entrypoint.sh + - backend_media:/app/media + command: /app/scripts/gunicorn.sh restart: on-failure depends_on: - - postgres - - rabbitmq - env_file: + postgres: &healthy-dependency + condition: service_healthy + rabbitmq: *healthy-dependency + env_file: &backend-env-files - .env/prod/pg - .env/prod/back - .env/prod/rmq @@ -23,29 +26,33 @@ services: - .env/prod/web bot: - build: - context: ./bot + build: ./bot depends_on: - - backend - volumes: - - ./bot:/app - environment: - PROD: 1 + backend: &started-dependency + condition: service_started + batcher: *started-dependency + memcached: *started-dependency env_file: - .env/prod/bot - .env/prod/web - command: /gunicorn.sh + command: /app/scripts/gunicorn.sh restart: on-failure memcached: image: memcached:latest - postgres: - image: postgres:14.5-alpine + postgres: &pg-conf + image: postgres:17-alpine volumes: - db_data:/var/lib/postgresql/data env_file: - .env/prod/pg + user: postgres + healthcheck: &pg-healthcheck + test: [ "CMD-SHELL", "pg_isready" ] + interval: 5s + timeout: 2s + retries: 5 nginx: build: @@ -55,61 +62,30 @@ services: - '80:80' - '443:443' depends_on: - - backend - - bot - - rabbitmq - - batcher + bot: *started-dependency volumes: - ./backend/static/:/static/ - - ./nginx/certbot/conf:/etc/letsencrypt - - ./nginx/certbot/www:/var/www/certbot restart: unless-stopped - command: '/bin/sh -c ''while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g "daemon off;"''' - - certbot: - container_name: certbot - image: certbot/certbot - volumes: - - ./nginx/certbot/conf:/etc/letsencrypt - - ./nginx/certbot/www:/var/www/certbot - restart: unless-stopped - entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" celery: build: ./backend - command: /start_celery.sh - volumes: - - ./backend:/app - env_file: - - .env/prod/back - - .env/prod/rmq - - .env/prod/pg - - .env/prod/bot + command: /app/scripts/start_celery.sh + env_file: *backend-env-files + environment: + - CELERY_WORKER_COUNT=10 depends_on: - - backend - - rabbitmq - - celery-beat: - build: ./backend - command: celery -A clicker beat -l info - volumes: - - ./backend:/app - env_file: - - .env/prod/back - - .env/prod/rmq - - .env/prod/pg - - .env/prod/bot - depends_on: - - backend - - rabbitmq + backend: *started-dependency rabbitmq: container_name: 'rabbitmq' image: 'rabbitmq:3-management-alpine' env_file: - .env/prod/rmq - ports: - - '15672:15672' + healthcheck: + <<: *pg-healthcheck + test: rabbitmq-diagnostics -q ping + interval: 10s + timeout: 2s redis: env_file: @@ -118,14 +94,16 @@ services: command: bash -c "redis-server --appendonly yes --requirepass $${REDIS_PASSWORD}" volumes: - redis_data:/data + healthcheck: + <<: *pg-healthcheck + test: "[ $$(redis-cli -a $${REDIS_PASSWORD} ping) = 'PONG' ]" batcher: - build: - context: ./batcher + build: ./batcher depends_on: - - redis - - batcher-postgres - - rabbitmq + redis: *healthy-dependency + batcher-postgres: *healthy-dependency + rabbitmq: *healthy-dependency env_file: - .env/prod/rmq - .env/prod/redis @@ -134,7 +112,7 @@ services: - .env/prod/bot batcher-postgres: - image: postgres:14.5-alpine + <<: *pg-conf volumes: - batcher_db_data:/var/lib/postgresql/data env_file: diff --git a/docker-compose.yml b/docker-compose.yml index fbf7993..0b75eb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,20 +2,20 @@ volumes: db_data: {} batcher_db_data: {} redis_data: {} + backend_media: {} services: backend: - build: - context: ./backend + build: ./backend depends_on: - - postgres - - rabbitmq + postgres: &healthy-dependency + condition: service_healthy + rabbitmq: *healthy-dependency volumes: - - ./backend:/app - command: /start.sh - entrypoint: /entrypoint.sh + - backend_media:/app/media + command: /app/scripts/start.sh restart: on-failure - env_file: + env_file: &backend-env-files - .env/dev/pg - .env/dev/back - .env/dev/rmq @@ -24,44 +24,31 @@ services: ports: - '8000:8000' - postgres: - image: postgres:14.5-alpine + postgres: &pg-conf + image: postgres:17-alpine volumes: - db_data:/var/lib/postgresql/data env_file: - .env/dev/pg ports: - '5432:5432' + user: postgres + healthcheck: &pg-healthcheck + test: [ "CMD-SHELL", "pg_isready" ] + interval: 5s + timeout: 2s + retries: 5 celery: build: ./backend - command: celery -A clicker worker -l info - volumes: - - ./backend:/app - env_file: - - .env/dev/back - - .env/dev/rmq - - .env/dev/pg - - .env/dev/bot - - .env/dev/web + command: /app/scripts/start_celery.sh + env_file: *backend-env-files + environment: + - CELERY_WORKER_COUNT=1 depends_on: - - backend - - rabbitmq - - celery-beat: - build: ./backend - command: celery -A clicker beat -l info - volumes: - - ./backend:/app - env_file: - - .env/dev/back - - .env/dev/rmq - - .env/dev/pg - - .env/dev/bot - - .env/dev/web - depends_on: - - backend - - rabbitmq + backend: + condition: service_started + rabbitmq: *healthy-dependency rabbitmq: container_name: 'rabbitmq' @@ -71,6 +58,11 @@ services: ports: - '5672:5672' - '15672:15672' + healthcheck: + <<: *pg-healthcheck + test: rabbitmq-diagnostics -q ping + interval: 10s + timeout: 2s redis: env_file: @@ -81,13 +73,16 @@ services: - '6379:6379' volumes: - redis_data:/data + healthcheck: + <<: *pg-healthcheck + test: "[ $$(redis-cli -a $$REDIS_PASSWORD ping) = 'PONG' ]" batcher: - build: - context: ./batcher + build: ./batcher depends_on: - - redis - - batcher-postgres + redis: *healthy-dependency + batcher-postgres: *healthy-dependency + rabbitmq: *healthy-dependency env_file: - .env/dev/rmq - .env/dev/redis @@ -99,7 +94,7 @@ services: - '8080:8080' batcher-postgres: - image: postgres:14.5-alpine + <<: *pg-conf volumes: - batcher_db_data:/var/lib/postgresql/data env_file: diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 0c5289e..0faa3c4 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -13,7 +13,11 @@ RUN npm run build # stage 2 - nginx FROM nginx:stable + COPY nginx/nginx.conf /etc/nginx/nginx.conf + +COPY backend/static /static + COPY --from=build-deps /app/dist/ /dist/ CMD ["nginx", "-g", "daemon off;"] diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 653d56b..9828dca 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -42,33 +42,11 @@ http { access_log /var/log/nginx/access.log upstreamlog; error_log /var/log/nginx/error.log; listen 80; -# listen 443 ssl http2; charset utf-8; -# server_name kyc_clicker.ru www.kyc_clicker.ru; root /dist/; index index.html; -# ssl_certificate /etc/letsencrypt/live/kyc_clicker.ru/fullchain.pem; -# ssl_certificate_key /etc/letsencrypt/live/kyc_clicker.ru/privkey.pem; - -# include /etc/letsencrypt/options-ssl-nginx.conf; -# ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - -# if ($server_port = 80) { -# set $https_redirect 1; -# } -# if ($host ~ '^www\.') { -# set $https_redirect 1; -# } -# if ($https_redirect = 1) { -# return 301 https://crowngame.ru$request_uri; -# } - -# location /.well-known/acme-challenge/ { -# root /var/www/certbot; -# } - # frontend location / { try_files $uri $uri/ @rewrites; @@ -79,7 +57,8 @@ http { } # batcher - location ~ ^/api/v1/(batch\-click|click|energy|coefficient)(/(.*))? { + location ~ ^/api/v1/batcher(/?.*) { + rewrite ^(/api/v1)/batcher(/?.*)$ $1/$2 last; proxy_pass http://batcher; proxy_pass_header Authorization; } @@ -98,29 +77,12 @@ http { proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Upgrade $http_upgrade; - proxy_cache_bypass $http_upgrade; - if ($uri ~* \.(?:ico|js|css|gif|jpe?g|png|webp)/?$) { - expires max; - add_header Pragma public; - add_header Cache-Control "public, must-revalidate, proxy-revalidate"; - } } # bot location ~ ^/bot { proxy_http_version 1.1; proxy_pass http://bot; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Port $server_port; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Upgrade $http_upgrade; - proxy_cache_bypass $http_upgrade; } # backend static