Enhance tg-index with performance, UI and monitoring features

.replit

@@ -0,0 +1,49 @@
+modules = ["python-3.12", "web", "nodejs-20"]
+[agent]
+expertMode = true
+
+[nix]
+channel = "stable-25_05"
+packages = ["cargo", "freetype", "lcms2", "libiconv", "libimagequant", "libjpeg", "libtiff", "libwebp", "libxcrypt", "openjpeg", "openssl", "rustc", "tcl", "tk", "zlib", "code-server", "nano", "opencode"]
+
+[userenv]
+
+[userenv.shared]
+API_ID = "964098"
+PORT = "5000"
+API_HASH = "3a40779521bc99b4c9753572ddd17ee7"
+SESSION_STRING = "1BVtsOKIBuzxzYSRe1YYf1IHe02W4iMZUHyMIR9rdLGjH6apna953BrsnCixhY9SG6gTxFRNwrlec96iA622GjQrcHNMLOmU76QKjk-XNP5PSMYIQf0-1N6TSVqe3XEp4n8ClXev53DCIR0dQyR5l4jZJivjL9gA-PjqLMuCfwSz6TiehA80xEP6lOxipGeivh0GOI8h2lIbKmLn65XKzp67JMTVwLHdpyCJBaa-c6dX60mWenhe6bxK2RWJjA2ZrKmMJxJAaCrRFxhHYYNPfJU9PjrS2sCBvkPmappc6FuNItWnDp-SjizkrNPxP30YAi1UC3lTeoaEYmMYRq-yYiIkxZzBRLCc="
+index_all = "true"
+INDEX_SETTINGS = "{\"index_all\":true,\"index_private\":true,\"index_group\":true,\"index_channel\":true,\"exclude_chats\":[],\"include_chats\":[]}"
+
+[workflows]
+runButton = "Project"
+
+[[workflows.workflow]]
+name = "Project"
+mode = "parallel"
+author = "agent"
+
+[[workflows.workflow.tasks]]
+task = "workflow.run"
+args = "Start application"
+
+[[workflows.workflow]]
+name = "Start application"
+author = "agent"
+
+[[workflows.workflow.tasks]]
+task = "shell.exec"
+args = "python repl-config/run-repl.py"
+waitForPort = 5000
+
+[workflows.workflow.metadata]
+outputType = "webview"
+
+[deployment]
+deploymentTarget = "vm"
+run = ["python", "repl-config/run-repl.py"]
+
+[[ports]]
+localPort = 5000
+externalPort = 80

README.md

@@ -1,64 +1,85 @@
-# Telegram Index
+# Telegram Index (Forked Version)
 
-> Python Web App which indexes a telegram channel(or a chat) and serves its files for download.
+An enhanced fork of [tg-index](https://github.com/odysseusmax/tg-index) with major performance, UI, and feature improvements.
 
-[![Open Source Love](https://badges.frapsoft.com/os/v1/open-source.png?v=103)](.) [![GPLv3 license](https://img.shields.io/badge/License-GPLv3-blue.svg)](LICENSE)
+## Changes Made(enchanced)
 
-## Highlights
+- **JWPlayer Video Player**: Custom control bar buttons, Material Symbols icons, direct file save
+- **UI Redesign**: DaisyUI, dark/light theme, mobile responsive
+- **Performance**: LRU cache (TTL), disk cache (mmap), back pressure, rate limiting, circuit breaker, exponential backoff
+- **Monitoring**: Health endpoint, response time tracking
+- **Bug Fixes**: JS scoping, chat_ids lookup, _me attribute
 
-- Index one or more telegram channels/chats.
-- View messages and media files on the browser.
-- Search through the channel/chat.
-- Download media files through browser/download managers.
+---
 
-## Deploy Guide
+A Python web application that indexes Telegram channels and chats, serving media files for browsing and download with a modern UI.
 
-- **Clone to local machine.**
+[![Open Source](https://badges.frapsoft.com/os/v1/open-source.png)](LICENSE) [![GPLv3](https://img.shields.io/badge/License-GPLv3-blue.svg)](LICENSE)
 
-```bash
-$ git clone https://github.com/odysseusmax/tg-index.git
+## Features
 
-$ cd tg-index
-```
+| Category | Feature |
+| -------- | -------- |
+| **Indexing** | Index multiple Telegram channels/chats, view messages and media in browser |
+| **Video Player** | JWPlayer with custom controls (Rewind 10s, Forward 10s, Download), Material Symbols icons |
+| **Audio** | Integrated audio player support |
+| **Search** | Full-text search through indexed content |
+| **UI** | DaisyUI redesign, dark/light theme toggle, mobile responsive |
+| **Downloads** | Direct file save via browser or download managers |
+| **Auth** | Optional username/password authentication |
 
-- **Create and activate virtual environment.**
+## Performance
 
-```bash
-$ python -m venv venv
+- LRU cache with TTL support
+- Disk cache with mmap for fast I/O
+- Back pressure controller (50 concurrent requests)
+- Rate limiting (100 requests/60s per IP)
+- Circuit breaker for downstream failures
+- Exponential backoff for Telegram FloodWait errors
+- Graceful shutdown handling
 
-$ source venv/bin/activate
-```
+## Monitoring
 
-- **Install dependencies.**
-
-```bash
-$ pip3 install -U -r requirements.txt
-```
+- Health metrics endpoint (`/health`)
+- Response time tracking (rolling average)
+- Status code distribution
+- Request ID tracking for debugging
 
-- **Environment Variables.**
+## Requirements
 
-| Variable Name                        | Value                                                                                                                                                          |
-| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `API_ID` (required)                  | Telegram api_id obtained from <https://my.telegram.org/apps>.                                                                                                  |
-| `API_HASH` (required)                | Telegram api_hash obtained from <https://my.telegram.org/apps>.                                                                                                |
-| `INDEX_SETTINGS` (required)          | See the below description.                                                                                                                                     |
-| `SESSION_STRING` (required)          | String obtained by running `$ python3 app/generate_session_string.py`. (Login with the telegram account which is a participant of the given channel (or chat). |
-| `PORT` (optional)                    | Port on which app should listen to, defaults to 8080.                                                                                                          |
-| `HOST` (optional)                    | Host name on which app should listen to, defaults to 0.0.0.0.                                                                                                  |
-| `DEBUG` (optional)                   | Give `true` to set logging level to debug, info by default.                                                                                                    |
-| `BLOCK_DOWNLOADS` (optional)         | Enable downloads or not. If any value is provided, downloads will be disabled.                                                                                 |
-| `RESULTS_PER_PAGE` (optional)        | Number of results to be returned per page defaults to 20.                                                                                                      |
-| `TGINDEX_USERNAME` (optional)        | Username for authentication, defaults to `''`.                                                                                                                 |
-| `PASSWORD` (optional)                | Password for authentication, defaults to `''`.                                                                                                                 |
-| `SHORT_URL_LEN` (optional)           | Url length for aliases                                                                                                                                         |
-| `SESSION_COOKIE_LIFETIME` (optional) | Number of minutes, for which authenticated session is valid for, after which user has to login again. defaults to 60.                                          |
-| `SECRET_KEY` (optional)              | 32 characters long string for signing the session cookies, required if authentication is enabled.                                                              |
+- Python 3.8+
+- Telegram API credentials from [my.telegram.org/apps](https://my.telegram.org/apps)
 
-- **Setting value for `INDEX_SETTINGS`**
+## Installation
 
-This is the general format, change the values of corresponding fields as your requirements. You can copy paste this as is to index all the channels available in your account.
+```bash
+git clone https://github.com/odysseusmax/tg-index.git
+cd tg-index
+python -m venv venv
+source venv/bin/activate
+pip3 install -U -r requirements.txt
+```
 
-**Remember to remove spaces.**
+## Configuration
+
+| Variable | Required | Description |
+| -------- | -------- | ---------- |
+| `API_ID` | Yes | Telegram API ID |
+| `API_HASH` | Yes | Telegram API hash |
+| `SESSION_STRING` | Yes | From `python3 app/generate_session_string.py` |
+| `INDEX_SETTINGS` | Yes | Indexing config (see below) |
+| `PORT` | No | Server port (default: 8080) |
+| `HOST` | No | Server host (default: 0.0.0.0) |
+| `DEBUG` | No | Set `true` for debug logging |
+| `RESULTS_PER_PAGE` | No | Results per page (default: 20) |
+| `TGINDEX_USERNAME` | No | Auth username |
+| `PASSWORD` | No | Auth password |
+| `SECRET_KEY` | No* | 32-char key for signed sessions (*required if auth enabled) |
+| `BLOCK_DOWNLOADS` | No | If set, disables downloads |
+| `SHORT_URL_LEN` | No | URL alias length |
+| `SESSION_COOKIE_LIFETIME` | No | Session validity in minutes (default: 60) |
+
+### INDEX_SETTINGS
 
 ```json
 {
@@ -71,31 +92,55 @@ This is the general format, change the values of corresponding fields as your re
 }
 ```
 
-> - `index_all` - Whether to consider all the chats associated with the telegram account. Value should either be `true` or `false`.
-> - `index_private` - Whether to index private chats. Only considered if `index_all` is set to `true`. Value should either be `true` or `false`.
-> - `index_group` - Whether to index group chats. Only considered if `index_all` is set to `true`. Value should either be `true` or `false`.
-> - `index_channel` - Whether to index channels. Only considered if `index_all` is set to `true`. Value should either be `true` or `false`.
-> - `exclude_chats` - An array/list of chat id's that should be ignored for indexing. Only considered if `index_all` is set to `true`.
-> - `include_chats` - An array/list of chat id's to index. Only considered if `index_all` is set to `false`.
+- `index_all`: Index all chats or use specific settings
+- `index_private`: Include private chats
+- `index_group`: Include group chats  
+- `index_channel`: Include channels
+- `exclude_chats`: Chat IDs to skip (when `index_all: true`)
+- `include_chats`: Specific chats to index (when `index_all: false`)
 
-- **Run app.**
+## Running
 
 ```bash
 python3 -m app
 ```
 
-## Deploy Guide (Repl.it)
+## Deployment
+
+### Docker
+
+```dockerfile
+FROM python:3.11-slim
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+COPY . .
+CMD ["python3", "-m", "app"]
+```
+
+### Repl.it
+
+See [replit-deploy-guide.md](./repl-config/replit-deploy-guide.md) for detailed setup instructions.
 
-A detailed and beginner friendly guide on how to deploy this project on a free instance of <https://repl.it> can be found [here](./repl-config/replit-deploy-guide.md).
+## Troubleshooting
 
-## Contributions
+| Issue | Solution |
+| ----- | -------- |
+| FloodWait errors | Exponential backoff handles automatically; reduce request frequency |
+| Slow playback | Enable disk cache in settings |
+| Rate limited | Check `security.py` configuration |
+| High memory | Adjust back pressure limits |
 
-Contributions are welcome.
+## What's New
 
-## Contact
+Recent major enhancements:
 
-You can contact me [@odysseusmax](https://tx.me/odysseusmax).
+- **JWPlayer Video Player**: Custom control bar buttons (Rewind 10s, Forward 10s, Download), Material Symbols icons, direct file save
+- **UI Redesign**: Complete DaisyUI template, dark/light theme toggle, enhanced thumbnails, mobile responsive
+- **Performance**: LRU cache with TTL, disk cache with mmap, back pressure (50 concurrent), rate limiting (100/min), circuit breaker, exponential backoff, graceful shutdown
+- **Monitoring**: Health endpoint, response time tracking, status code distribution, request ID tracking
+- **Bug Fixes**: JS scoping (var→let), chat_ids lookup, _me attribute, runtime errors
 
 ## License
 
-Code released under [The GNU General Public License](LICENSE).
+Released under [GNU General Public License](LICENSE).

app/backpressure.py

@@ -0,0 +1,88 @@
+import asyncio
+import logging
+from functools import wraps
+from typing import Callable, Any, TypeVar
+
+from aiohttp import web
+
+from app.config import results_per_page
+
+log = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class OptimisticLock:
+    def __init__(self):
+        self._locks: dict = {}
+        self._lock = asyncio.Lock()
+
+    async def get_lock(self, key: str) -> asyncio.Lock:
+        async with self._lock:
+            if key not in self._locks:
+                self._locks[key] = asyncio.Lock()
+            return self._locks[key]
+
+    async def acquire(self, key: str) -> bool:
+        lock = await self.get_lock(key)
+        return await lock.acquire()
+
+    def release(self, key: str) -> None:
+        if key in self._locks:
+            self._locks[key].release()
+
+
+class BackPressureController:
+    def __init__(self, max_concurrent: int = 50):
+        self._max_concurrent = max_concurrent
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+        self._active_tasks: int = 0
+        self._waiting_tasks: int = 0
+        self._total_served: int = 0
+
+    async def __aenter__(self) -> None:
+        self._waiting_tasks += 1
+        await self._semaphore.acquire()
+        self._waiting_tasks -= 1
+        self._active_tasks += 1
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
+        self._active_tasks -= 1
+        self._total_served += 1
+        self._semaphore.release()
+
+    @property
+    def active(self) -> int:
+        return self._active_tasks
+
+    @property
+    def waiting(self) -> int:
+        return self._waiting_tasks
+
+
+class Debouncer:
+    def __init__(self, delay: float = 0.3):
+        self._delay = delay
+        self._timers = {}
+        self._lock = asyncio.Lock()
+
+    async def debounce(self, key: str, coro_fn: Callable) -> None:
+        async with self._lock:
+            if key in self._timers:
+                self._timers[key].cancel()
+
+        async def run():
+            async with self._lock:
+                if key in self._timers:
+                    del self._timers[key]
+            await coro_fn()
+
+        loop = asyncio.get_event_loop()
+        loop.call_later(self._delay, lambda: asyncio.create_task(run()))
+
+    async def cancel(self, key: str) -> None:
+        async with self._lock:
+            if key in self._timers:
+                self._timers[key].cancel()
+                del self._timers[key]