diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml new file mode 100644 index 0000000..a12a548 --- /dev/null +++ b/.github/workflows/develop.yml @@ -0,0 +1,85 @@ +name: CI/CD - develop + +on: + push: + branches: + - develop + pull_request: + branches: + - develop + +env: + IMAGE: ghcr.io/$(echo $GITHUB_REPOSITORY | tr '[A-Z]' '[a-z]')/aimingmed-ai-backend + +jobs: + + build: + name: Build Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main + - name: Log in to GitHub Packages + run: echo ${GITHUB_TOKEN} | docker login -u ${GITHUB_ACTOR} --password-stdin ghcr.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Pull image + run: | + docker pull ${{ env.IMAGE }}:latest || true + - name: Build image + run: | + docker build \ + --cache-from ${{ env.IMAGE }}:latest \ + --tag ${{ env.IMAGE }}:latest \ + --file ./app/backend/Dockerfile.prod \ + "./app/backend" + - name: Push image + run: | + docker push ${{ env.IMAGE }}:latest + + test: + name: Test Docker Image + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main + - name: Log in to GitHub Packages + run: echo ${GITHUB_TOKEN} | docker login -u ${GITHUB_ACTOR} --password-stdin ghcr.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Pull image + run: | + docker pull ${{ env.IMAGE }}:latest || true + - name: Build image + run: | + docker build \ + --cache-from ${{ env.IMAGE }}:latest \ + --tag ${{ env.IMAGE }}:latest \ + --file ./app/backend/Dockerfile.prod \ + "./app/backend" + - name: Run container + run: | + docker run \ + -d \ + --name backend \ + -e PORT=8765 \ + -e ENVIRONMENT=dev \ + -e TESTING=0 \ + -p 8004:8765 \ + ${{ env.IMAGE }}:latest + - name: Pytest + run: docker exec backend pipenv run python -m pytest . + # - name: Flake8 + # run: docker exec backend pipenv run python -m flake8 . + # - name: Black + # run: docker exec backend pipenv run python -m black . --check + # - name: isort + # run: docker exec backend pipenv run python -m isort . --check-only \ No newline at end of file diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..22d660a --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.8" diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..1fc96b8 --- /dev/null +++ b/app/README.md @@ -0,0 +1,50 @@ +# How to work with this app repository + +Build the images: + +```bash +docker compose up --build -d +``` + +I + +# Run the tests for backend: + +```bash +docker compose exec backend pipenv run python -m pytest --disable-warnings --cov="." +``` + +Lint: + +```bash +docker compose exec backend pipenv run flake8 . +``` + +Run Black and isort with check options: + +```bash +docker compose exec backend pipenv run black . --check +docker compose exec backend pipenv run isort . --check-only +``` + +Make code changes with Black and isort: + +```bash +docker compose exec backend pipenv run black . +docker compose exec backend pipenv run isort . +``` + +# Postgres + +Want to access the database via psql? + +```bash +docker compose exec -it database psql -U postgres +``` + +Then, you can connect to the database and run SQL queries. For example: + +```sql +# \c web_dev +# \dt +``` diff --git a/app/backend/Dockerfile b/app/backend/Dockerfile index 57b7b59..5ccef9c 100644 --- a/app/backend/Dockerfile +++ b/app/backend/Dockerfile @@ -10,7 +10,7 @@ ENV PYTHONUNBUFFERED 1 # install system dependencies RUN apt-get update \ - && apt-get -y install build-essential netcat-traditional gcc postgresql \ + && apt-get -y install build-essential netcat-traditional gcc \ && apt-get clean # install python dependencies diff --git a/app/backend/Dockerfile.prod b/app/backend/Dockerfile.prod new file mode 100644 index 0000000..2a63aba --- /dev/null +++ b/app/backend/Dockerfile.prod @@ -0,0 +1,84 @@ +########### +# BUILDER # +########### + +# pull official base image +FROM python:3.11-slim AS builder + + +# set working directory +WORKDIR /usr/src/app + +# set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# install system dependencies +RUN apt-get update \ + && apt-get -y install build-essential netcat-traditional gcc \ + && apt-get clean + +# install python dependencies +RUN pip install --upgrade pip setuptools wheel -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -i https://pypi.tuna.tsinghua.edu.cn/simple pipenv +RUN pip install --no-cache-dir --find-links=/usr/src/app/wheels pipenv +COPY ./Pipfile . +RUN pipenv install --deploy + +# add app +COPY . /usr/src/app +RUN pipenv run pip install black==23.12.1 flake8==7.0.0 isort==5.13.2 +# RUN pipenv run flake8 . +# RUN pipenv run black --exclude=migrations . --check +# RUN pipenv run isort . --check-only + +######### +# FINAL # +######### + +# pull official base image +FROM python:3.11-slim + +# create directory for the app user +RUN mkdir -p /home/app + +# create the app user +RUN addgroup --system app && adduser --system --group app + + +# create the appropriate directories +ENV HOME=/home/app +ENV APP_HOME=/home/app/backend +RUN mkdir $APP_HOME +WORKDIR $APP_HOME + +# set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV ENVIRONMENT=prod +ENV TESTING=0 + +# install system dependencies +RUN apt-get update \ + && apt-get -y install build-essential netcat-traditional gcc \ + && apt-get clean + +# install python dependencies +COPY --from=builder /usr/src/app/wheels /wheels +COPY --from=builder /usr/src/app/Pipfile . +RUN pip install --upgrade pip setuptools wheel -i https://pypi.tuna.tsinghua.edu.cn/simple +RUN pip install --no-cache /wheels/* +RUN pipenv install --deploy +RUN pipenv run pip install "uvicorn[standard]==0.26.0" + +# add app +COPY . $APP_HOME + +# chown all the files to the app user +RUN chown -R app:app $APP_HOME + +# change to the app user +USER app + +# run gunicorn +CMD pipenv run gunicorn --bind 0.0.0.0:$PORT backend.main:app -k uvicorn.workers.UvicornWorker \ No newline at end of file diff --git a/app/backend/Pipfile b/app/backend/Pipfile index f9b8531..46c5de6 100644 --- a/app/backend/Pipfile +++ b/app/backend/Pipfile @@ -4,13 +4,13 @@ verify_ssl = true name = "pypi" [packages] -fastapi = "*" -pydantic = "*" -uvicorn = "*" +fastapi = "==0.115.9" +starlette = "==0.45.3" +uvicorn = "==0.26.0" pydantic-settings = "==2.1.0" -python-decouple = "*" +gunicorn = "==21.0.1" +python-decouple = "==3.8" pyyaml = "==6.0.1" -pip = "==24.0.0" docker = "*" chromadb = "*" sentence-transformers = "*" diff --git a/app/backend/Pipfile.lock b/app/backend/Pipfile.lock index 4508d20..7b7772f 100644 --- a/app/backend/Pipfile.lock +++ b/app/backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "05c19a932e8fd2238dab9a8ea53ef955cea73016b068887c8a410b33ec4c5ce8" + "sha256": "22c986576b1b18ff70ae1fefc20957d1758705ed23f84388376db2a6a478a99e" }, "pipfile-spec": 6, "requires": { @@ -271,16 +271,16 @@ }, "chromadb": { "hashes": [ - "sha256:4a604f212312c5c10b7470104de45123b75eb7239e1599b22127aa9a414ca78e", - "sha256:51d6adf1859774535f7ab6c6b5492f514970aa26a3e636861dbc7766418d6c08", - "sha256:55d514189998734470f700a9d7094ee578949e5df9f891a177e690413071f234", - "sha256:582c5307a03a766bf321a023ce9e3c34b0cb7427658e102aacf6742c9d900a5d", - "sha256:9d7a0c59c5ce95c78f711cf435526b6fdde5158d015132c04af8cb3b04c742d0", - "sha256:a29e12c4ac0fdfb12b3ee22ea3c66d2fa4e3516ee9fe3cad0f44b3086d1c50c4" + "sha256:32daa01014acf98570eeb31e966b641a4d79a2fdc1f750caa5dfc1ba24d9991c", + "sha256:37e8071e3bc40f0a67730f9483ca1d3e8a67d32a3409cd1beaacfb76336eb98c", + "sha256:844a2a3bd624149093be84b9c3e2b070fa21da129c8563c790be64c4bf0d9f5f", + "sha256:9c4e72dba33b6fd2da55f044498b298169724cae5113538fa18b3458427ecea8", + "sha256:ee927adfe618e170320b6da25b39a01282142350de70b9b76ab98aa7af7d2f34", + "sha256:f6789b573f510815218b25027f8471b6c132acc2f2d2a53ff42e3e76d9d248ac" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==1.0.3" + "version": "==1.0.4" }, "click": { "hashes": [ @@ -435,6 +435,15 @@ "markers": "python_version >= '3.9'", "version": "==1.71.0" }, + "gunicorn": { + "hashes": [ + "sha256:949880781d74f55eda34eb1a552f9c83db6edb42f2bd4f87c09e2a66b13922ea", + "sha256:b18fa5a9b22becdffc29d2586b914225a69624bb3e3a064cb04decfb2f34bfe8" + ], + "index": "pypi", + "markers": "python_version >= '3.5'", + "version": "==21.0.1" + }, "h11": { "hashes": [ "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", @@ -730,11 +739,11 @@ }, "langsmith": { "hashes": [ - "sha256:060956aaed5f391a85829daa0c220b5e07b2e7dd5d33be4b92f280672be984f7", - "sha256:0bdeda73cf723cbcde1cab0f3459f7e5d5748db28a33bf9f6bdc0e2f4fe0ee1e" + "sha256:4666595207131d7f8d83418e54dc86c05e28562e5c997633e7c33fc18f9aeb89", + "sha256:54ac8815514af52d9c801ad7970086693667e266bf1db90fc453c1759e8407cd" ], "markers": "python_version >= '3.9' and python_version < '4.0'", - "version": "==0.3.27" + "version": "==0.3.28" }, "markdown-it-py": { "hashes": [ @@ -1276,15 +1285,6 @@ "markers": "python_version >= '3.9'", "version": "==11.1.0" }, - "pip": { - "hashes": [ - "sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - "sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==24.0" - }, "posthog": { "hashes": [ "sha256:1ac0305ab6c54a80c4a82c137231f17616bef007bbf474d1a529cda032d808eb", @@ -1330,7 +1330,6 @@ "sha256:7471657138c16adad9322fe3070c0116dd6c3ad8d649300e3cbdfe91f4db4ec3", "sha256:a082753436a07f9ba1289c6ffa01cd93db3548776088aa917cc43b63f68fa60f" ], - "index": "pypi", "markers": "python_version >= '3.9'", "version": "==2.11.3" }, @@ -2030,6 +2029,7 @@ "sha256:2cbcba2a75806f8a41c722141486f37c28e30a0921c5f6fe4346cb0dcee1302f", "sha256:dfb6d332576f136ec740296c7e8bb8c8a7125044e7c6da30744718880cdd059d" ], + "index": "pypi", "markers": "python_version >= '3.9'", "version": "==0.45.3" }, @@ -2194,12 +2194,12 @@ "standard" ], "hashes": [ - "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4", - "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9" + "sha256:48bfd350fce3c5c57af5fb4995fded8fb50da3b4feb543eb18ad7e0d54589602", + "sha256:cdb58ef6b8188c6c174994b2b1ba2150a9a8ae7ea5fb2f1b856b94a815d6071d" ], "index": "pypi", - "markers": "python_version >= '3.9'", - "version": "==0.34.0" + "markers": "python_version >= '3.8'", + "version": "==0.26.0" }, "uvloop": { "hashes": [ diff --git a/app/backend/api/ping.py b/app/backend/api/ping.py index 3ec5e35..fbd8ae5 100644 --- a/app/backend/api/ping.py +++ b/app/backend/api/ping.py @@ -1,7 +1,6 @@ from fastapi import APIRouter, Depends -from config import get_settings, Settings - +from config import Settings, get_settings router = APIRouter() @@ -11,5 +10,5 @@ async def pong(settings: Settings = Depends(get_settings)): return { "ping": "pong!", "environment": settings.environment, - "testing": settings.testing - } \ No newline at end of file + "testing": settings.testing, + } diff --git a/app/backend/tests/api/test_chatbot.py b/app/backend/tests/api/test_chatbot.py index d92b208..6c40686 100644 --- a/app/backend/tests/api/test_chatbot.py +++ b/app/backend/tests/api/test_chatbot.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import AsyncMock from fastapi import WebSocket, WebSocketDisconnect -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) -from app.backend.api.chatbot import websocket_endpoint, manager, llm_chat +from api.chatbot import websocket_endpoint, manager, llm_chat diff --git a/app/backend/tests/api/test_utils.py b/app/backend/tests/api/test_utils.py index 1ecf54a..796fcda 100644 --- a/app/backend/tests/api/test_utils.py +++ b/app/backend/tests/api/test_utils.py @@ -4,9 +4,9 @@ import unittest from unittest.mock import AsyncMock, MagicMock from fastapi import WebSocket -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) -from app.backend.api.utils import ConnectionManager +from api.utils import ConnectionManager class TestConnectionManager(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): diff --git a/app/backend/tests/conftest.py b/app/backend/tests/conftest.py index 38cd119..ef86afb 100644 --- a/app/backend/tests/conftest.py +++ b/app/backend/tests/conftest.py @@ -1,10 +1,8 @@ -import os - import pytest from starlette.testclient import TestClient -import main from config import get_settings, Settings +from main import create_application def get_settings_override(): @@ -14,10 +12,10 @@ def get_settings_override(): @pytest.fixture(scope="module") def test_app(): # set up - main.app.dependency_overrides[get_settings] = get_settings_override - with TestClient(main.app) as test_client: - + app = create_application() + app.dependency_overrides[get_settings] = get_settings_override + with TestClient(app) as test_client: # testing yield test_client - # tear down \ No newline at end of file + # tear down diff --git a/app/docker-compose.yml b/app/docker-compose.yml index fdcd2c4..3aee244 100644 --- a/app/docker-compose.yml +++ b/app/docker-compose.yml @@ -13,7 +13,6 @@ services: build: context: ./backend dockerfile: Dockerfile - platform: linux/amd64 command: pipenv run uvicorn main:app --reload --workers 1 --host 0.0.0.0 --port 8000 volumes: - ./backend:/usr/src/app @@ -21,4 +20,4 @@ services: - "8004:8000" environment: - ENVIRONMENT=dev - - TESTING=0 \ No newline at end of file + - TESTING=0