From 9181cbc072d64def17b0232f8616b5f687686828 Mon Sep 17 00:00:00 2001 From: Shivam Menda Date: Wed, 18 Jun 2025 01:05:22 +0530 Subject: [PATCH 1/9] Added RabbitMQ initial setup --- .gitignore | 179 +----------------- .../app/core/orchestration/queue_manager.py | 48 +++-- backend/requirements.txt | Bin 10994 -> 11020 bytes 3 files changed, 38 insertions(+), 189 deletions(-) diff --git a/.gitignore b/.gitignore index 7be9213d..f514b74c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,177 +1,2 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ - -lib/ -!frontend/src/lib - -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# PyPI configuration file -.pypirc - -# Local Netlify folder -.netlify +# Created by venv; see https://docs.python.org/3/library/venv.html +* diff --git a/backend/app/core/orchestration/queue_manager.py b/backend/app/core/orchestration/queue_manager.py index caef6c53..d16098f8 100644 --- a/backend/app/core/orchestration/queue_manager.py +++ b/backend/app/core/orchestration/queue_manager.py @@ -3,6 +3,9 @@ from typing import Dict, Any, Callable, Optional from datetime import datetime from enum import Enum +import pika +import time +import json logger = logging.getLogger(__name__) @@ -12,27 +15,44 @@ class QueuePriority(str, Enum): LOW = "low" class AsyncQueueManager: - """AsyncIO-based queue manager for agent orchestration""" + """Queue manager for agent orchestration""" def __init__(self): + self.connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) + self.channel = self.connection.channel() self.queues = { - QueuePriority.HIGH: asyncio.Queue(), - QueuePriority.MEDIUM: asyncio.Queue(), - QueuePriority.LOW: asyncio.Queue() + QueuePriority.HIGH: 'high_task_queue', + QueuePriority.MEDIUM: 'medium_task_queue', + QueuePriority.LOW: 'low_task_queue' } self.handlers: Dict[str, Callable] = {} self.running = False self.worker_tasks = [] + def callback(ch, method, properties, body): + print(f" [x] Received {body.decode()}") + time.sleep(body.count(b'.')) + print(" [x] Done") + ch.basic_ack(delivery_tag=method.delivery_tag) + async def start(self, num_workers: int = 3): """Start the queue processing workers""" self.running = True - for i in range(num_workers): - task = asyncio.create_task(self._worker(f"worker-{i}")) - self.worker_tasks.append(task) - - logger.info(f"Started {num_workers} queue workers") + # for i in range(num_workers): + # task = asyncio.create_task(self._worker(f"worker-{i}")) + # self.worker_tasks.append(task) + + logger.info("Setting QOS") + self.channel.basic_qos(prefetch_count=1) + logger.info("QOS set") + logger.info("Starting Queues") + for i in self.queues.values(): + self.channel.queue_declare(queue=i, durable=True) + logger.info(f"Queue {i} started") + logger.info("Starting Channel") + self.channel.start_consuming() + logger.info(f"Started channel {self.channel.channel_number}") async def stop(self): """Stop the queue processing""" @@ -44,6 +64,9 @@ async def stop(self): await asyncio.gather(*self.worker_tasks, return_exceptions=True) logger.info("Stopped all queue workers") + self.channel.stop_consuming() + self.connection.close() + logger.info('Stopped Queue') async def enqueue(self, message: Dict[str, Any], @@ -60,9 +83,10 @@ async def enqueue(self, "priority": priority, "data": message } - - await self.queues[priority].put(queue_item) - logger.debug(f"Enqueued message {queue_item['id']} with priority {priority}") + json_message = json.dumps(queue_item) + # await self.queues[priority].put(queue_item) + self.channel.basic_publish(exchange='', routing_key=f'{self.queues[priority]}', body=json_message) + logger.info(f"Enqueued message {queue_item['id']} with priority {priority}") def register_handler(self, message_type: str, handler: Callable): """Register a handler for a specific message type""" diff --git a/backend/requirements.txt b/backend/requirements.txt index 5d1fdbbd22fddf9cb4ab48eecb824b645e431083..3a95b3c8dcdd6c8e7f61fcb1fe6411eb6b42f361 100644 GIT binary patch delta 32 kcmewq+7q^6nS^jALpDPqgDnsmGUzcF1F_NO`4YiG0G|N}8vp Date: Sat, 28 Jun 2025 11:34:55 +0530 Subject: [PATCH 2/9] Update requirements.txt --- backend/requirements.txt | 275 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) diff --git a/backend/requirements.txt b/backend/requirements.txt index e69de29b..2dac24c6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -0,0 +1,275 @@ +absl-py==2.1.0 +aiohappyeyeballs==2.4.4 +aiohttp==3.11.13 +aiosignal==1.3.2 +alembic==1.15.1 +annotated-types==0.7.0 +anyio==4.6.0 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-lru==2.0.4 +async-timeout==5.0.1 +asyncpg==0.30.0 +attrs==24.2.0 +babel==2.16.0 +bcrypt==4.3.0 +beautifulsoup4==4.12.3 +bidict==0.23.1 +bleach==6.1.0 +blinker==1.9.0 +blis==0.7.11 +cachetools==5.5.0 +catalogue==2.0.10 +certifi==2024.8.30 +cffi==1.17.1 +charset-normalizer==3.4.0 +click==8.1.8 +cloudpathlib==0.20.0 +cobble==0.1.4 +colorama==0.4.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.0 +cycler==0.12.1 +cymem==2.0.10 +datasets==3.1.0 +debugpy==1.8.7 +decorator==5.1.1 +defusedxml==0.7.1 +deprecation==2.1.0 +dill==0.3.8 +distlib==0.3.9 +dnspython==2.7.0 +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl#sha256=86cc141f63942d4b2c5fcee06630fd6f904788d2f0ab005cce45aadb8fb73889 +eventlet==0.38.2 +exceptiongroup==1.2.2 +executing==2.1.0 +fastapi==0.115.11 +fastjsonschema==2.20.0 +filelock==3.16.1 +flashtext==2.7 +Flask==3.1.0 +Flask-Cors==5.0.0 +Flask-SocketIO==5.5.1 +flatbuffers==24.3.25 +fonttools==4.54.1 +fqdn==1.5.1 +frozenlist==1.5.0 +fsspec==2024.9.0 +future==0.18.3 +fuzzywuzzy==0.18.0 +gast==0.4.0 +gensim==4.3.3 +google-ai-generativelanguage==0.6.10 +google-api-core==2.22.0 +google-api-python-client==2.113.0 +google-auth==2.35.0 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-generativeai==0.8.3 +google-pasta==0.2.0 +googleapis-common-protos==1.65.0 +gotrue==2.11.4 +greenlet==3.1.1 +grpcio==1.67.1 +grpcio-status==1.67.1 +h11==0.14.0 +h2==4.2.0 +h5py==3.12.1 +hpack==4.1.0 +httpcore==1.0.6 +httplib2==0.22.0 +httpx==0.27.2 +huggingface-hub==0.27.1 +hyperframe==6.1.0 +idna==3.10 +ipykernel==6.29.5 +ipython==8.28.0 +ipywidgets==8.1.5 +isoduration==20.11.0 +itsdangerous==2.2.0 +jedi==0.19.1 +Jinja2==3.1.4 +joblib==1.2.0 +json5==0.9.25 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +jupyter==1.1.1 +jupyter-console==6.6.3 +jupyter-events==0.10.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.7.2 +jupyter_server==2.14.2 +jupyter_server_terminals==0.5.3 +jupyterlab==4.2.5 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_widgets==3.0.13 +kaggle==1.6.17 +kagglehub==0.3.1 +keras==2.13.1 +keras-tuner==1.4.7 +kiwisolver==1.4.7 +kt-legacy==1.0.5 +langcodes==3.5.0 +language_data==1.3.0 +libclang==18.1.1 +Mako==1.3.9 +mammoth==1.9.0 +marisa-trie==1.2.1 +Markdown==3.7 +markdown-it-py==3.0.0 +MarkupSafe==3.0.1 +matplotlib==3.9.2 +matplotlib-inline==0.1.7 +mdurl==0.1.2 +mediawikiapi==1.2 +mistune==3.0.2 +ml-dtypes==0.4.1 +mpmath==1.3.0 +multidict==6.1.0 +multiprocess==0.70.16 +murmurhash==1.0.11 +namex==0.0.8 +nbclient==0.10.0 +nbconvert==7.16.4 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.1 +nltk==3.9.1 +notebook==7.2.2 +notebook_shim==0.2.4 +numpy==1.24.3 +oauth2client==4.1.3 +oauthlib==3.2.2 +opencv-python==4.10.0.84 +opt_einsum==3.4.0 +optree==0.13.0 +overrides==7.7.0 +packaging==24.1 +pandas==2.2.3 +pandocfilters==1.5.1 +parso==0.8.4 +patsy==0.5.6 +pgmpy==0.1.26 +pillow==10.4.0 +pke @ git+https://github.com/boudinfl/pke.git@69871ffdb720b83df23684fea53ec8776fd87e63 +platformdirs==4.3.6 +postgrest==0.19.3 +preshed==3.0.9 +prometheus_client==0.21.0 +prompt_toolkit==3.0.48 +propcache==0.2.1 +proto-plus==1.25.0 +protobuf==4.25.5 +psutil==6.0.0 +psycopg2==2.9.10 +pure_eval==0.2.3 +pyarrow==18.1.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 +pycparser==2.22 +pydantic==2.9.2 +pydantic_core==2.23.4 +Pygments==2.18.0 +PyMuPDF==1.25.1 +pyparsing==3.1.4 +python-dateutil==2.8.2 +python-dotenv==1.0.1 +python-engineio==4.11.2 +python-json-logger==2.0.7 +python-slugify==8.0.4 +python-socketio==5.12.1 +pytz==2022.7.1 +pywin32==308 +pywinpty==2.0.13 +PyYAML==6.0.2 +pyzmq==26.2.0 +realtime==2.4.1 +referencing==0.35.1 +regex==2024.11.6 +requests==2.32.3 +requests-oauthlib==2.0.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==13.9.2 +rpds-py==0.20.0 +rsa==4.9 +safetensors==0.5.1 +scikit-learn==1.5.2 +scipy==1.13.1 +seaborn==0.13.2 +Send2Trash==1.8.3 +sense2vec==2.0.2 +sentencepiece==0.2.0 +shellingham==1.5.4 +simple-websocket==1.1.0 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +soupsieve==2.6 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +SQLAlchemy==2.0.38 +srsly==2.5.0 +stack-data==0.6.3 +starlette==0.46.1 +statsmodels==0.14.4 +storage3==0.11.3 +StrEnum==0.4.15 +strsim==0.0.3 +supabase==2.13.0 +supafunc==0.9.3 +sympy==1.13.1 +tensorboard==2.13.0 +tensorboard-data-server==0.7.2 +tensorflow==2.13.1 +tensorflow-estimator==2.13.0 +tensorflow-intel==2.13.1 +tensorflow-io-gcs-filesystem==0.31.0 +termcolor==2.5.0 +terminado==0.18.1 +text-unidecode==1.3 +thinc==8.2.5 +threadpoolctl==3.5.0 +tinycss2==1.3.0 +tokenizers==0.20.3 +tomli==2.0.2 +torch==2.5.1 +tornado==6.4.1 +tqdm==4.66.5 +traitlets==5.14.3 +transformers==4.46.1 +typer==0.15.1 +types-python-dateutil==2.9.0.20241003 +typing_extensions==4.12.2 +tzdata==2024.2 +Unidecode==1.3.0 +uri-template==1.3.0 +uritemplate==4.1.1 +urllib3==2.2.3 +uv==0.5.9 +uvicorn==0.34.0 +virtualenv==20.29.2 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.8.0 +webencodings==0.5.1 +websocket-client==1.8.0 +websockets==14.2 +Werkzeug==3.1.3 +whisper==1.1.10 +widgetsnbextension==4.0.13 +wrapt==1.16.0 +wsproto==1.2.0 +xgboost==2.1.2 +xxhash==3.5.0 +yarl==1.18.3 +weaviate-client==4.9.6 From e8f0cac95b1d3f703bfbe8c009f71bda28d95949 Mon Sep 17 00:00:00 2001 From: Shivam Menda Date: Sat, 28 Jun 2025 14:48:49 +0530 Subject: [PATCH 3/9] Add Async Queue manager final implementation --- .../app/core/orchestration/queue_manager.py | 128 +++++++----------- backend/main.py | 6 +- backend/requirements.txt | 1 + 3 files changed, 58 insertions(+), 77 deletions(-) diff --git a/backend/app/core/orchestration/queue_manager.py b/backend/app/core/orchestration/queue_manager.py index d16098f8..61e82157 100644 --- a/backend/app/core/orchestration/queue_manager.py +++ b/backend/app/core/orchestration/queue_manager.py @@ -3,8 +3,7 @@ from typing import Dict, Any, Callable, Optional from datetime import datetime from enum import Enum -import pika -import time +import aio_pika import json logger = logging.getLogger(__name__) @@ -18,8 +17,6 @@ class AsyncQueueManager: """Queue manager for agent orchestration""" def __init__(self): - self.connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) - self.channel = self.connection.channel() self.queues = { QueuePriority.HIGH: 'high_task_queue', QueuePriority.MEDIUM: 'medium_task_queue', @@ -28,31 +25,26 @@ def __init__(self): self.handlers: Dict[str, Callable] = {} self.running = False self.worker_tasks = [] + self.connection: Optional[aio_pika.RobustConnection] = None + self.channel: Optional[aio_pika.abc.AbstractChannel] = None - def callback(ch, method, properties, body): - print(f" [x] Received {body.decode()}") - time.sleep(body.count(b'.')) - print(" [x] Done") - ch.basic_ack(delivery_tag=method.delivery_tag) + async def connect(self): + self.connection = await aio_pika.connect_robust("amqp://guest:guest@localhost/") + self.channel = await self.connection.channel() + # Declare queues + for queue_name in self.queues.values(): + await self.channel.declare_queue(queue_name, durable=True) async def start(self, num_workers: int = 3): """Start the queue processing workers""" + await self.connect() self.running = True - # for i in range(num_workers): - # task = asyncio.create_task(self._worker(f"worker-{i}")) - # self.worker_tasks.append(task) - - logger.info("Setting QOS") - self.channel.basic_qos(prefetch_count=1) - logger.info("QOS set") - logger.info("Starting Queues") - for i in self.queues.values(): - self.channel.queue_declare(queue=i, durable=True) - logger.info(f"Queue {i} started") - logger.info("Starting Channel") - self.channel.start_consuming() - logger.info(f"Started channel {self.channel.channel_number}") + for i in range(num_workers): + task = asyncio.create_task(self._worker(f"worker-{i}")) + self.worker_tasks.append(task) + + logger.info(f"Started {num_workers} async queue workers") async def stop(self): """Stop the queue processing""" @@ -63,10 +55,11 @@ async def stop(self): task.cancel() await asyncio.gather(*self.worker_tasks, return_exceptions=True) - logger.info("Stopped all queue workers") - self.channel.stop_consuming() - self.connection.close() - logger.info('Stopped Queue') + if self.channel: + await self.channel.close() + if self.connection: + await self.connection.close() + logger.info("Stopped all queue workers and closed connection") async def enqueue(self, message: Dict[str, Any], @@ -79,13 +72,14 @@ async def enqueue(self, queue_item = { "id": message.get("id", f"msg_{datetime.now().timestamp()}"), - "timestamp": datetime.now().isoformat(), "priority": priority, "data": message } - json_message = json.dumps(queue_item) - # await self.queues[priority].put(queue_item) - self.channel.basic_publish(exchange='', routing_key=f'{self.queues[priority]}', body=json_message) + json_message = json.dumps(queue_item).encode() + await self.channel.default_exchange.publish( + aio_pika.Message(body=json_message), + routing_key=self.queues[priority] + ) logger.info(f"Enqueued message {queue_item['id']} with priority {priority}") def register_handler(self, message_type: str, handler: Callable): @@ -96,50 +90,29 @@ def register_handler(self, message_type: str, handler: Callable): async def _worker(self, worker_name: str): """Worker coroutine to process queue items""" logger.info(f"Started queue worker: {worker_name}") - + # Each worker listens to all queues by priority + queues = [ + await self.channel.declare_queue(self.queues[priority], durable=True) + for priority in [QueuePriority.HIGH, QueuePriority.MEDIUM, QueuePriority.LOW] + ] while self.running: - try: - # Process queues by priority - item = await self._get_next_item() - - if item: - await self._process_item(item, worker_name) - else: - # No items available, wait a bit - await asyncio.sleep(0.1) - - except asyncio.CancelledError: - logger.info(f"Worker {worker_name} cancelled") - break - except (ConnectionError, TimeoutError) as e: - logger.error(f"Connection error in worker {worker_name}: {str(e)}") - await asyncio.sleep(5) # Longer pause for connection issues - except Exception as e: - logger.error(f"Unexpected error in worker {worker_name}: {str(e)}") - await asyncio.sleep(1) # Brief pause on error - - async def _get_next_item(self) -> Optional[Dict[str, Any]]: - """Get the next item from queues (priority-based)""" - - # Try high priority first - try: - return self.queues[QueuePriority.HIGH].get_nowait() - except asyncio.QueueEmpty: - pass - - # Then medium priority - try: - return self.queues[QueuePriority.MEDIUM].get_nowait() - except asyncio.QueueEmpty: - pass - - # Finally low priority - try: - return self.queues[QueuePriority.LOW].get_nowait() - except asyncio.QueueEmpty: - pass - - return None + for queue in queues: + try: + message = await queue.get(no_ack=False, fail=False) + if message: + try: + item = json.loads(message.body.decode()) + await self._process_item(item, worker_name) + await message.ack() + except Exception as e: + logger.error(f"Error processing message: {e}") + await message.nack(requeue=False) + except asyncio.CancelledError: + logger.info(f"Worker {worker_name} cancelled") + return + except Exception as e: + logger.error(f"Worker {worker_name} error: {e}") + await asyncio.sleep(0.1) async def _process_item(self, item: Dict[str, Any], worker_name: str): """Process a queue item""" @@ -151,9 +124,12 @@ async def _process_item(self, item: Dict[str, Any], worker_name: str): if handler: logger.debug(f"Worker {worker_name} processing {item['id']} (type: {message_type})") - await handler(message_data) + if asyncio.iscoroutinefunction(handler): + await handler(message_data) + else: + handler(message_data) else: logger.warning(f"No handler found for message type: {message_type}") except Exception as e: - logger.error(f"Error processing item {item['id']}: {str(e)}") + logger.error(f"Error processing item {item.get('id', 'unknown')}: {str(e)}") diff --git a/backend/main.py b/backend/main.py index 77a69440..42109bdd 100644 --- a/backend/main.py +++ b/backend/main.py @@ -114,7 +114,11 @@ async def health_check(): "status": "healthy", "services": { "weaviate": "ready" if weaviate_ready else "not_ready", - "discord_bot": "running" if app_instance.discord_bot and not app_instance.discord_bot.is_closed() else "stopped" + "discord_bot": ( + "running" + if app_instance.discord_bot and not app_instance.discord_bot.is_closed() + else "stopped" + ) } } except Exception as e: diff --git a/backend/requirements.txt b/backend/requirements.txt index 2dac24c6..2f6548b0 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -273,3 +273,4 @@ xgboost==2.1.2 xxhash==3.5.0 yarl==1.18.3 weaviate-client==4.9.6 +aio-pika==9.5.5 \ No newline at end of file From adf586b90daba323e52844bcd965021b054f0d50 Mon Sep 17 00:00:00 2001 From: Shivam Menda Date: Tue, 1 Jul 2025 00:05:57 +0530 Subject: [PATCH 4/9] Revert main.py and requirements.txt --- .DS_Store | Bin 0 -> 6148 bytes .gitignore | 179 +++++++++++++++++++++++++++++++++++++++++++++++- backend/main.py | 39 +++-------- pyvenv.cfg | 5 ++ 4 files changed, 190 insertions(+), 33 deletions(-) create mode 100644 .DS_Store create mode 100644 pyvenv.cfg diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..bd3818cfbe1a3e8062e2d7cf1b0a2f35c2b743d1 GIT binary patch literal 6148 zcmeHKOHRW;4E6LQQV>#Ch~;nr)UMftDy%{31)xnSNJ&#FV4d4=65s;Bo+aWC@H}Is zsgfdA2#_t=Z{o2h&Px^7L}Ugxai6G5L;;ks(SfNE-e;{yLp;1`+!+%}XrIpLGMQGQ z)$tn{;CI)dm?m^f3)tfQ^@rmuopZ^}-s^LHRGLN6D9J|H4cf;~dv6<$hwCEaZ${29 zN_W2R;W+&G&QV)NGrC5FGm4YxGG=q}epeS$t}SRE>|A|3aBY>d(rj7TS8?UKwqH)T zE)``!8BhlPDg)@*EWwsXwaS1ppbQiY@b|$&8AHL`qx*EgxDf!@g4+q!JWFtn6$}M) zkJy12p9=J;Mn(+d)8V%g7YgPceL5K#K8)Pi$b@3t-EqEk;bcOOYLx+HV3mQUzwC1V zKUzQkuO{i0GN26nD+Wv}8b$*=lJwTf!*Q>*&{HT2$K@W26f9gTMl84DU8ocIEiZtf TVD1qfi2f09G^kMqew2YvuZ&ZN literal 0 HcmV?d00001 diff --git a/.gitignore b/.gitignore index f514b74c..716b5d33 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,177 @@ -# Created by venv; see https://docs.python.org/3/library/venv.html -* +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ + +lib/ +!frontend/src/lib + +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# PyPI configuration file +.pypirc + +# Local Netlify folder +.netlify \ No newline at end of file diff --git a/backend/main.py b/backend/main.py index 42109bdd..94840427 100644 --- a/backend/main.py +++ b/backend/main.py @@ -6,13 +6,13 @@ import uvicorn from fastapi import FastAPI, Response -from app.api.v1.auth import router as auth_router +from app.api.router import api_router from app.core.config import settings from app.core.orchestration.agent_coordinator import AgentCoordinator from app.core.orchestration.queue_manager import AsyncQueueManager -from app.db.weaviate.weaviate_client import get_weaviate_client -from bots.discord.discord_bot import DiscordBot -from bots.discord.discord_cogs import DevRelCommands +from app.database.weaviate.client import get_weaviate_client +from integrations.discord.bot import DiscordBot +from integrations.discord.cogs import DevRelCommands logging.basicConfig( level=logging.INFO, @@ -91,6 +91,7 @@ async def lifespan(app: FastAPI): """ Lifespan manager for the FastAPI application. Handles startup and shutdown events. """ + app.state.app_instance = app_instance await app_instance.start_background_tasks() yield await app_instance.stop_background_tasks() @@ -103,32 +104,8 @@ async def favicon(): """Return empty favicon to prevent 404 logs""" return Response(status_code=204) -@api.get("/health") -async def health_check(): - """Health check endpoint to verify services are running""" - try: - async with get_weaviate_client() as client: - weaviate_ready = await client.is_ready() - - return { - "status": "healthy", - "services": { - "weaviate": "ready" if weaviate_ready else "not_ready", - "discord_bot": ( - "running" - if app_instance.discord_bot and not app_instance.discord_bot.is_closed() - else "stopped" - ) - } - } - except Exception as e: - logger.error(f"Health check failed: {e}") - return { - "status": "unhealthy", - "error": str(e) - } - -api.include_router(auth_router, prefix="/v1/auth", tags=["Authentication"]) + +api.include_router(api_router) if __name__ == "__main__": @@ -147,4 +124,4 @@ async def health_check(): host="0.0.0.0", port=8000, reload=True - ) + ) \ No newline at end of file diff --git a/pyvenv.cfg b/pyvenv.cfg new file mode 100644 index 00000000..74d5aa35 --- /dev/null +++ b/pyvenv.cfg @@ -0,0 +1,5 @@ +home = /opt/homebrew/opt/python@3.13/bin +include-system-site-packages = false +version = 3.13.1 +executable = /opt/homebrew/Cellar/python@3.13/3.13.1/Frameworks/Python.framework/Versions/3.13/bin/python3.13 +command = /opt/homebrew/opt/python@3.13/bin/python3.13 -m venv /Users/shivam/Desktop/Devr.AI/devrai From 2f6a50b9c6b3d0cbcb6ae909e4ef1b53c3b151c6 Mon Sep 17 00:00:00 2001 From: Shivam Menda Date: Tue, 1 Jul 2025 00:09:29 +0530 Subject: [PATCH 5/9] Remove pyvenv.cfg --- pyvenv.cfg | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 pyvenv.cfg diff --git a/pyvenv.cfg b/pyvenv.cfg deleted file mode 100644 index 74d5aa35..00000000 --- a/pyvenv.cfg +++ /dev/null @@ -1,5 +0,0 @@ -home = /opt/homebrew/opt/python@3.13/bin -include-system-site-packages = false -version = 3.13.1 -executable = /opt/homebrew/Cellar/python@3.13/3.13.1/Frameworks/Python.framework/Versions/3.13/bin/python3.13 -command = /opt/homebrew/opt/python@3.13/bin/python3.13 -m venv /Users/shivam/Desktop/Devr.AI/devrai From a549cda742d3c5ef83264f825078de0c6550d3bb Mon Sep 17 00:00:00 2001 From: Shivam Menda <74780977+ShivamMenda@users.noreply.github.com> Date: Tue, 1 Jul 2025 00:23:59 +0530 Subject: [PATCH 6/9] Update backend/app/core/orchestration/queue_manager.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../app/core/orchestration/queue_manager.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/backend/app/core/orchestration/queue_manager.py b/backend/app/core/orchestration/queue_manager.py index 61e82157..2cd23558 100644 --- a/backend/app/core/orchestration/queue_manager.py +++ b/backend/app/core/orchestration/queue_manager.py @@ -28,12 +28,20 @@ def __init__(self): self.connection: Optional[aio_pika.RobustConnection] = None self.channel: Optional[aio_pika.abc.AbstractChannel] = None +from app.core.config import settings + async def connect(self): - self.connection = await aio_pika.connect_robust("amqp://guest:guest@localhost/") - self.channel = await self.connection.channel() - # Declare queues - for queue_name in self.queues.values(): - await self.channel.declare_queue(queue_name, durable=True) + try: + rabbitmq_url = getattr(settings, 'rabbitmq_url', 'amqp://guest:guest@localhost/') + self.connection = await aio_pika.connect_robust(rabbitmq_url) + self.channel = await self.connection.channel() + # Declare queues + for queue_name in self.queues.values(): + await self.channel.declare_queue(queue_name, durable=True) + logger.info("Successfully connected to RabbitMQ") + except Exception as e: + logger.error(f"Failed to connect to RabbitMQ: {e}") + raise async def start(self, num_workers: int = 3): """Start the queue processing workers""" From c861ce062ad7e8bf5b802106c76f0571ffff2db4 Mon Sep 17 00:00:00 2001 From: Shivam Menda Date: Tue, 1 Jul 2025 00:39:47 +0530 Subject: [PATCH 7/9] Indendation fix --- backend/app/core/orchestration/queue_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/app/core/orchestration/queue_manager.py b/backend/app/core/orchestration/queue_manager.py index 2cd23558..346bc9a0 100644 --- a/backend/app/core/orchestration/queue_manager.py +++ b/backend/app/core/orchestration/queue_manager.py @@ -5,6 +5,7 @@ from enum import Enum import aio_pika import json +from app.core.config import settings logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ def __init__(self): self.connection: Optional[aio_pika.RobustConnection] = None self.channel: Optional[aio_pika.abc.AbstractChannel] = None -from app.core.config import settings + async def connect(self): try: From a544334dd41975300142becfb687002fbffea6ed Mon Sep 17 00:00:00 2001 From: Shivam Menda Date: Wed, 2 Jul 2025 21:19:44 +0530 Subject: [PATCH 8/9] Update settings and lockfile --- backend/app/core/config/settings.py | 3 ++ poetry.lock | 54 +++++++++++++++++++++++++++-- pyproject.toml | 1 + 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/backend/app/core/config/settings.py b/backend/app/core/config/settings.py index d3a531cb..5ccb79e3 100644 --- a/backend/app/core/config/settings.py +++ b/backend/app/core/config/settings.py @@ -31,6 +31,9 @@ class Settings(BaseSettings): classification_agent_model: str = "gemini-1.5-flash" agent_timeout: int = 30 max_retries: int = 3 + + # RabbitMQ configuration + rabbitmq_url: str = "" # Backend URL backend_url: str = "" diff --git a/poetry.lock b/poetry.lock index 3672fff6..9edd8f09 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,23 @@ # This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +[[package]] +name = "aio-pika" +version = "9.5.5" +description = "Wrapper around the aiormq for asyncio and humans" +optional = false +python-versions = "<4.0,>=3.9" +groups = ["main"] +files = [ + {file = "aio_pika-9.5.5-py3-none-any.whl", hash = "sha256:94e0ac3666398d6a28b0c3b530c1febf4c6d4ececb345620727cfd7bfe1c02e0"}, + {file = "aio_pika-9.5.5.tar.gz", hash = "sha256:3d2f25838860fa7e209e21fc95555f558401f9b49a832897419489f1c9e1d6a4"}, +] + +[package.dependencies] +aiormq = ">=6.8,<6.9" +exceptiongroup = ">=1,<2" +typing-extensions = {version = "*", markers = "python_version < \"3.10\""} +yarl = "*" + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -121,6 +139,22 @@ yarl = ">=1.17.0,<2.0" [package.extras] speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "brotlicffi ; platform_python_implementation != \"CPython\""] +[[package]] +name = "aiormq" +version = "6.8.1" +description = "Pure python AMQP asynchronous client library" +optional = false +python-versions = "<4.0,>=3.8" +groups = ["main"] +files = [ + {file = "aiormq-6.8.1-py3-none-any.whl", hash = "sha256:5da896c8624193708f9409ffad0b20395010e2747f22aa4150593837f40aa017"}, + {file = "aiormq-6.8.1.tar.gz", hash = "sha256:a964ab09634be1da1f9298ce225b310859763d5cf83ef3a7eae1a6dc6bd1da1a"}, +] + +[package.dependencies] +pamqp = "3.3.0" +yarl = "*" + [[package]] name = "aiosignal" version = "1.3.2" @@ -622,11 +656,11 @@ description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" groups = ["main", "dev"] -markers = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, ] +markers = {dev = "python_version < \"3.11\""} [package.dependencies] typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} @@ -2492,6 +2526,22 @@ files = [ {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] +[[package]] +name = "pamqp" +version = "3.3.0" +description = "RabbitMQ Focused AMQP low-level library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "pamqp-3.3.0-py2.py3-none-any.whl", hash = "sha256:c901a684794157ae39b52cbf700db8c9aae7a470f13528b9d7b4e5f7202f8eb0"}, + {file = "pamqp-3.3.0.tar.gz", hash = "sha256:40b8795bd4efcf2b0f8821c1de83d12ca16d5760f4507836267fd7a02b06763b"}, +] + +[package.extras] +codegen = ["lxml", "requests", "yapf"] +testing = ["coverage", "flake8", "flake8-comprehensions", "flake8-deprecated", "flake8-import-order", "flake8-print", "flake8-quotes", "flake8-rst-docstrings", "flake8-tuple", "yapf"] + [[package]] name = "pathspec" version = "0.12.1" @@ -4933,4 +4983,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.9, <4.0" -content-hash = "e522e012ca2cf1b0e41d143829af4aeaee30ce66714794ab238d644134e6d28c" +content-hash = "a6e367d06a6d8b8506aa3056187f3fe7613d4753ab2403abf39a1be65ad1638c" diff --git a/pyproject.toml b/pyproject.toml index e5861d13..f39a3a26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "weaviate-client (>=4.15.0,<5.0.0)", "langchain-google-genai (>=2.1.5,<3.0.0)", "python-dotenv (>=1.1.1,<2.0.0)", + "aio-pika (>=9.5.5,<10.0.0)", ] [tool.poetry] From 4b85c8e77cd1ef04999f9cda42b7ce22ee94fffa Mon Sep 17 00:00:00 2001 From: Shivam Menda Date: Wed, 2 Jul 2025 21:33:47 +0530 Subject: [PATCH 9/9] Added uvicorn --- poetry.lock | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 9edd8f09..d30dc75c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -491,6 +491,38 @@ files = [ {file = "charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63"}, ] +[[package]] +name = "click" +version = "8.1.8" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "click" +version = "8.2.1" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, + {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "colorama" version = "0.4.6" @@ -4407,6 +4439,26 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uvicorn" +version = "0.35.0" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "uvicorn-0.35.0-py3-none-any.whl", hash = "sha256:197535216b25ff9b785e29a0b79199f55222193d47f820816e7da751e9bc8d4a"}, + {file = "uvicorn-0.35.0.tar.gz", hash = "sha256:bc662f087f7cf2ce11a1d7fd70b90c9f98ef2e2831556dd078d131b96cc94a01"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] + [[package]] name = "validators" version = "0.34.0" @@ -4983,4 +5035,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.9, <4.0" -content-hash = "a6e367d06a6d8b8506aa3056187f3fe7613d4753ab2403abf39a1be65ad1638c" +content-hash = "f00e5f86781e9cbfb6a4e28aa95c1ddd2923123f7e8bbbfdc814a1c099690aa9" diff --git a/pyproject.toml b/pyproject.toml index f39a3a26..2124898e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "langchain-google-genai (>=2.1.5,<3.0.0)", "python-dotenv (>=1.1.1,<2.0.0)", "aio-pika (>=9.5.5,<10.0.0)", + "uvicorn (>=0.35.0,<0.36.0)", ] [tool.poetry]