From f2aa382511efba5e3a8d631f0a8b5d79bc2a953c Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+vitalii-t12@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:31:13 +0200 Subject: [PATCH 01/11] Cerviguard serving (#308) * chore: inc version * rm: cerviguard image processor plugin * chore: inc version --- .../cerviguard/cerviguard_constants.py | 6 - .../cerviguard/cerviguard_image_processor.py | 210 ----------- .../business/cerviguard/local_serving_api.py | 241 +++++++++++-- .../cerviguard/cerviguard_image_analyzer.py | 336 ++++++++++++++++++ ver.py | 2 +- 5 files changed, 558 insertions(+), 237 deletions(-) delete mode 100644 extensions/business/cerviguard/cerviguard_constants.py delete mode 100644 extensions/business/cerviguard/cerviguard_image_processor.py create mode 100644 extensions/serving/cerviguard/cerviguard_image_analyzer.py diff --git a/extensions/business/cerviguard/cerviguard_constants.py b/extensions/business/cerviguard/cerviguard_constants.py deleted file mode 100644 index 603c1928..00000000 --- a/extensions/business/cerviguard/cerviguard_constants.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Shared constants for CerviGuard components. -""" - -REQUEST_PAYLOAD_TYPE = 'cerviguard_request' -RESULT_PAYLOAD_TYPE = 'cerviguard_result' diff --git a/extensions/business/cerviguard/cerviguard_image_processor.py b/extensions/business/cerviguard/cerviguard_image_processor.py deleted file mode 100644 index b53b4c15..00000000 --- a/extensions/business/cerviguard/cerviguard_image_processor.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -CerviGuard Image Processor Plugin - -Processes payloads that enter the loopback queue (pushed by LOCAL_SERVING_API) -and emits structured result payloads back into that same queue. The API plugin -can then pick up the results via the standard Data API, so no shared-memory -access is required between plugins. -""" - -from naeural_core.business.base import BasePluginExecutor as BasePlugin -from extensions.business.cerviguard.cerviguard_constants import ( - REQUEST_PAYLOAD_TYPE, - RESULT_PAYLOAD_TYPE, -) - -__VER__ = '0.2.0' - -_CONFIG = { - **BasePlugin.CONFIG, - - 'ALLOW_EMPTY_INPUTS': True, - 'PROCESS_DELAY': 0.1, # Fast processing for responsiveness - 'IS_LOOPBACK_PLUGIN': True, # Results are pushed back into the queue - - 'VALIDATION_RULES': { - **BasePlugin.CONFIG['VALIDATION_RULES'], - }, -} - - -class CerviguardImageProcessorPlugin(BasePlugin): - """ - Image processor plugin for CerviGuard WAR application. - - Reads images from loopback queue, processes them (currently just dimensions), - and emits result payloads back into the same queue. - """ - - CONFIG = _CONFIG - - def on_init(self): - super(CerviguardImageProcessorPlugin, self).on_init() - self._processed_count = 0 - self.P("CerviGuard Image Processor initialized", color='g') - self.P(" Ready to process images from loopback queue", color='g') - return - - def _decode_base64_image(self, image_data: str): - """ - Decode base64 encoded image to a numpy array (returns None on failure). - """ - try: - # Handle data URL format (data:image/png;base64,...) - if ',' in image_data: - image_data = image_data.split(',', 1)[1] - - img_bytes = self.base64_to_bytes(image_data) - img = self.PIL.Image.open(self.BytesIO(img_bytes)) - img_array = self.np.array(img) - return img_array - except Exception as e: - self.P(f"Error decoding image: {e}", color='r') - return None - - def _process_image_dimensions(self, img_array, request_id: str, metadata: dict) -> dict: - """ - Process image to extract dimensions (current implementation) - - In the future, this will call the CerviGuard AI model for analysis - - Parameters - ---------- - img_array : np.ndarray - Image as numpy array - request_id : str - Unique request identifier - metadata : dict - Additional metadata from request - - Returns - ------- - dict - Processing results - """ - if img_array is None or len(img_array.shape) < 2: - return { - 'status': 'error', - 'error': 'Invalid image data' - } - - # Extract dimensions - height, width = img_array.shape[:2] - channels = img_array.shape[2] if len(img_array.shape) > 2 else 1 - - # Calculate size info - total_pixels = height * width - size_mb = img_array.nbytes / (1024 * 1024) - - # Prepare result - result = { - 'status': 'completed', - 'request_id': request_id, - 'image_info': { - 'width': int(width), - 'height': int(height), - 'channels': int(channels), - 'total_pixels': int(total_pixels), - 'size_mb': round(size_mb, 3), - 'dtype': str(img_array.dtype), - 'shape': list(img_array.shape), - }, - 'processed_at': self.time(), - 'processor_version': __VER__, - 'metadata': metadata, - } - - # TODO: Future enhancement - call AI model - # if self.has_ai_engine(): - # ai_results = self.predict(img_array) - # result['ai_analysis'] = ai_results - - return result - - def _get_payload_field(self, data: dict, key: str, default=None): - if key in data: - return data[key] - key_upper = key.upper() - if key_upper in data: - return data[key_upper] - return default - - def process(self): - """ - Main processing loop - reads from loopback queue and processes images. - """ - payloads = self.dataapi_struct_datas(full=False, as_list=True) - - if not payloads: - # No data to process - return None - - self.P(f"Retrieved {len(payloads)} payload(s) from loopback queue", color='b') - - for payload in payloads: - if not isinstance(payload, dict): - self.P(f"Skipping non-dict payload from loopback: {payload}", color='y') - continue - self._process_payload(payload) - - return None - - - def _process_payload(self, data: dict): - """ - Handle a single payload emitted through the loopback queue. - """ - payload_type = self._get_payload_field(data, 'payload_type') - if payload_type != REQUEST_PAYLOAD_TYPE: - self.P(f"Ignoring payload type '{payload_type}'", color='c') - return - - # Extract request info - request_id = self._get_payload_field(data, 'request_id') - image_data = self._get_payload_field(data, 'image_data') - metadata = self._get_payload_field(data, 'metadata', {}) or {} - - if not request_id: - self.P("Received data without request_id, ignoring", color='y') - return - - if not image_data: - self._emit_error(request_id, 'Missing image data') - return - - self.P(f"Processing request {request_id} (keys={list(data.keys())})", color='b') - - # Decode image - img_array = self._decode_base64_image(image_data) - - if img_array is None: - self._emit_error(request_id, 'Failed to decode image') - return - - # Process image (get dimensions, later AI analysis) - result = self._process_image_dimensions(img_array, request_id, metadata) - - # Emit result payload back into loopback queue - self.add_payload_by_fields( - payload_type=RESULT_PAYLOAD_TYPE, - request_id=request_id, - result=result, - ) - - self._processed_count += 1 - self.P(f"Completed processing request {request_id} (total: {self._processed_count})", color='g') - return - - def _emit_error(self, request_id, message): - self.P(f"Request {request_id} failed: {message}", color='r') - error_payload = { - 'status': 'error', - 'error': message, - 'request_id': request_id, - } - self.add_payload_by_fields( - payload_type=RESULT_PAYLOAD_TYPE, - request_id=request_id, - result=error_payload, - ) - return diff --git a/extensions/business/cerviguard/local_serving_api.py b/extensions/business/cerviguard/local_serving_api.py index 149b7500..dfd139c9 100644 --- a/extensions/business/cerviguard/local_serving_api.py +++ b/extensions/business/cerviguard/local_serving_api.py @@ -23,10 +23,6 @@ """ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin -from extensions.business.cerviguard.cerviguard_constants import ( - REQUEST_PAYLOAD_TYPE, - RESULT_PAYLOAD_TYPE, -) __VER__ = '0.1.0' @@ -55,6 +51,9 @@ 'PROCESS_DELAY': 0, 'RESULT_CACHE_TTL': 300, + # AI Engine configuration for image analysis + 'AI_ENGINE': 'CERVIGUARD_IMAGE_ANALYZER', # Serving plugin to use + 'VALIDATION_RULES': { **FastApiWebAppPlugin.CONFIG['VALIDATION_RULES'], 'RESULT_CACHE_TTL': { @@ -88,9 +87,10 @@ def on_init(self): self._data_buffer = [] self._results_cache = {} self._last_result_cleanup = self.time() - self.P("Local Serving API initialized - Loopback mode enabled") - self.P(f"Server will be accessible only on localhost (no tunnel)") - self.P(f"Loopback key: loopback_dct_{self._stream_id}") + self.P("Local Serving API initialized - Loopback mode enabled", color='g') + self.P(f" Server accessible only on localhost (no tunnel)", color='g') + self.P(f" AI Engine: {self.cfg_ai_engine}", color='g') + self.P(f" Loopback key: loopback_dct_{self._stream_id}", color='g') return def _get_payload_field(self, data: dict, key: str, default=None): @@ -219,6 +219,114 @@ def process_image(self, image_data: str, metadata: dict = None): # ========== CERVIGUARD WAR ENDPOINTS ========== + @FastApiWebAppPlugin.endpoint(method="post") + def predict(self, image_data: str, metadata: dict = None): + """ + Simple /predict endpoint for image analysis + + Simplified endpoint that accepts an image and returns a request ID. + This is the main endpoint for the cerviguard flow: + 1. Receives base64 image + 2. Adds to loopback queue via add_payload_by_fields + 3. Serving plugin processes the image + 4. Results cached for polling + + Parameters + ---------- + image_data : str + Base64 encoded image (supports data URLs) + metadata : dict, optional + Additional metadata + + Returns + ------- + dict + Request ID and status for polling + """ + # Generate unique request ID + request_id = self.uuid() + + self.P(f"[Predict] Received image, request_id: {request_id}", color='b') + + # Validate image data + if not image_data or len(image_data) < 100: + return { + "status": "error", + "error": "Invalid or missing image data", + "message": "Image data must be base64 encoded" + } + + # Track request + self._data_buffer.append({ + "request_id": request_id, + "type": "prediction", + "submitted_at": self.time(), + "metadata": metadata or {} + }) + + # STEP 3: Send to loopback queue via add_payload_by_fields + # Because IS_LOOPBACK_PLUGIN=True, this writes to loopback_dct_{stream_id} queue + self.add_payload_by_fields( + request_id=request_id, + image_data=image_data, + metadata=metadata or {}, + type="prediction", + submitted_at=self.time() + ) + + self.P(f"[Predict] Image added to loopback queue: {request_id}", color='g') + + return { + "status": "submitted", + "request_id": request_id, + "message": "Image queued for analysis", + "poll_endpoint": f"/get_result?request_id={request_id}" + } + + @FastApiWebAppPlugin.endpoint(method="get") + def get_result(self, request_id: str): + """ + Get prediction result for /predict endpoint + + Poll this endpoint to retrieve the processing result. + """ + if not request_id: + return { + "status": "error", + "error": "Missing request_id parameter" + } + + self.P(f"[Predict] Result requested for: {request_id}", color='b') + + result = self._results_cache.get(request_id) + + if result is None: + submitted = any( + item.get('request_id') == request_id + for item in self._data_buffer + ) + + if submitted: + return { + "status": "processing", + "request_id": request_id, + "message": "Image is still being processed, please poll again" + } + else: + return { + "status": "not_found", + "request_id": request_id, + "error": "Request ID not found" + } + + self.P(f"[Predict] Returning result for: {request_id}", color='g') + + return { + "status": "completed", + "request_id": request_id, + "result": result['result'] + } + @FastApiWebAppPlugin.endpoint(method="post") def cerviguard_submit_image(self, image_data: str, metadata: dict = None): """ @@ -262,9 +370,7 @@ def cerviguard_submit_image(self, image_data: str, metadata: dict = None): }) # Send to loopback queue - # This will be picked up by CerviguardImageProcessorPlugin self.add_payload_by_fields( - payload_type=REQUEST_PAYLOAD_TYPE, request_id=request_id, image_data=image_data, metadata=metadata or {}, @@ -426,38 +532,133 @@ def batch_process(self, items: list): def process(self): """ - Main process loop - can be used for periodic tasks + Main process loop: + 1. Read struct_data from pipeline (contains image requests from loopback) + 2. Read inferences from serving plugin (which has already processed them) + 3. Match inferences to requests by index + 4. Cache results for retrieval via API """ self._cleanup_result_cache() self._maybe_trim_buffer() + # Read struct_data from pipeline (raw payloads) payloads = self.dataapi_struct_datas(full=False, as_list=True) if not payloads: return None - for payload in payloads: - self._handle_loopback_payload(payload) + # Read inferences that serving plugin already produced + # The serving plugin processes the data automatically via the pipeline + inferences = self.dataapi_struct_data_inferences(how='list') + + if not inferences: + self.P(f"No inferences available for {len(payloads)} payload(s)", color='y') + return None + + self.P(f"Processing {len(payloads)} payload(s) with {len(inferences)} inference(s)", color='b') + + # Match payloads with inferences (they should be in same order) + for idx, payload in enumerate(payloads): + inference = inferences[idx] if idx < len(inferences) else None + self._process_loopback_payload(payload, inference) return None - def _handle_loopback_payload(self, payload): + def _process_loopback_payload(self, payload, inference): + """ + Process a single payload from loopback queue with its corresponding inference: + 1. Extract request info from payload + 2. Extract inference result from serving plugin + 3. Cache result for API retrieval + + Parameters + ---------- + payload : dict + The original payload with request_id, image_data, metadata + inference : dict + The inference result from the serving plugin (already processed) + """ if not isinstance(payload, dict): return - payload_type = self._get_payload_field(payload, 'payload_type') - if payload_type != RESULT_PAYLOAD_TYPE: + # Extract request info from payload + request_id = self._get_payload_field(payload, 'request_id') + metadata = self._get_payload_field(payload, 'metadata', {}) or {} + + if not request_id: + self.P("Received payload without request_id, ignoring", color='y') return - request_id = self._get_payload_field(payload, 'request_id') - result = self._get_payload_field(payload, 'result') - if not request_id or result is None: + if inference is None: + self.P(f"No inference available for request {request_id}", color='y') + self._cache_error_result(request_id, 'No inference result available') return + self.P(f"[CerviGuard] Processing inference for request {request_id}", color='b') + + try: + # The serving plugin returns inferences in a specific format + # For CERVIGUARD_IMAGE_ANALYZER, it returns: {'status': 'completed', 'data': {...}} + + if not isinstance(inference, dict): + self.P(f"Unexpected inference format: {type(inference)}", color='r') + self._cache_error_result(request_id, 'Invalid inference result format') + return + + # Extract the inference data + # Inference can be the result dict directly or wrapped + inference_data = inference.get('data', inference) if 'data' in inference else inference + + # Check status + status = inference_data.get('status', 'unknown') + + if status == 'error': + error_msg = inference_data.get('error', 'Unknown error') + self._cache_error_result(request_id, error_msg) + return + + # Success - extract image info + image_info = inference_data.get('image_info', {}) + + final_result = { + 'status': 'completed', + 'request_id': request_id, + 'image_info': image_info, + 'processed_at': inference_data.get('processed_at', self.time()), + 'processor_version': inference_data.get('processor_version', 'unknown'), + 'metadata': metadata, + } + + # Cache the result + self._results_cache[request_id] = { + 'result': final_result, + 'stored_at': self.time(), + } + + self.P(f"[CerviGuard] Cached result for request {request_id}", color='g') + + except Exception as e: + self.P(f"Error processing request {request_id}: {e}", color='r') + import traceback + self.P(traceback.format_exc(), color='r') + self._cache_error_result(request_id, f'Processing error: {str(e)}') + + return + + def _cache_error_result(self, request_id: str, error_message: str): + """Cache an error result for a request""" + self.P(f"[CerviGuard] Caching error for request {request_id}: {error_message}", color='r') + + error_result = { + 'status': 'error', + 'error': error_message, + 'request_id': request_id, + 'processed_at': self.time(), + } + self._results_cache[request_id] = { - 'result': result, + 'result': error_result, 'stored_at': self.time(), } - self.P(f"[CerviGuard] Cached result for request {request_id}", color='g') return def _cleanup_result_cache(self): diff --git a/extensions/serving/cerviguard/cerviguard_image_analyzer.py b/extensions/serving/cerviguard/cerviguard_image_analyzer.py new file mode 100644 index 00000000..28204143 --- /dev/null +++ b/extensions/serving/cerviguard/cerviguard_image_analyzer.py @@ -0,0 +1,336 @@ +""" +CerviGuard Image Analyzer - Serving Plugin + +A serving plugin that analyzes cervical images from base64 encoded data. +Currently extracts image dimensions and properties as a mockup for future +AI-based cervical cancer detection models. + +This serving plugin runs in an isolated process and provides: +- Base64 image decoding +- Image dimension extraction +- Format and color space analysis +- Future placeholder for AI model inference + +Usage in pipeline: +{ + "PLUGINS": [ + { + "SIGNATURE": "A_SIMPLE_PLUGIN", + "INSTANCES": [ + { + "INSTANCE_ID": "cerviguard_01", + "AI_ENGINE": "CERVIGUARD_IMAGE_ANALYZER" + } + ] + } + ] +} +""" + +from naeural_core.serving.base import ModelServingProcess as BaseServingProcess + +import base64 +from PIL import Image +from io import BytesIO + +__VER__ = '0.1.0' + +_CONFIG = { + **BaseServingProcess.CONFIG, + + # Accept STRUCT_DATA input (base64 encoded images) + "PICKED_INPUT": "STRUCT_DATA", + + # Allow running without input for initialization + "RUNS_ON_EMPTY_INPUT": False, + + 'VALIDATION_RULES': { + **BaseServingProcess.CONFIG['VALIDATION_RULES'], + }, +} + + +class CerviguardImageAnalyzer(BaseServingProcess): + """ + Serving plugin for CerviGuard cervical image analysis. + + Processes base64 encoded images and extracts dimensional and + color information. Designed to be extended with AI models for + cervical cancer detection. + """ + + CONFIG = _CONFIG + + def on_init(self): + """ + Initialize the serving plugin. + Called once during startup. + """ + super(CerviguardImageAnalyzer, self).on_init() + self._processed_count = 0 + self.P("CerviGuard Image Analyzer initialized", color='g') + self.P(f" Version: {__VER__}", color='g') + self.P(f" Accepts STRUCT_DATA input (base64 images)", color='g') + return + + def _decode_base64_image(self, image_data): + """ + Decode base64 encoded image to numpy array. + + Parameters + ---------- + image_data : str or dict + Base64 encoded image string or dict containing 'image_data' key + + Returns + ------- + np.ndarray or None + Decoded image as numpy array, or None if decoding fails + """ + try: + # Handle dict input with 'image_data' key + if isinstance(image_data, dict): + if 'image_data' in image_data: + image_data = image_data['image_data'] + elif 'IMAGE_DATA' in image_data: + image_data = image_data['IMAGE_DATA'] + else: + self.P("Dict input missing 'image_data' key", color='r') + return None + + # Handle data URL format (data:image/png;base64,...) + if isinstance(image_data, str) and ',' in image_data: + image_data = image_data.split(',', 1)[1] + + # Decode base64 to bytes + img_bytes = base64.b64decode(image_data) + + # Convert to PIL Image + img = Image.open(BytesIO(img_bytes)) + + # Convert to numpy array + img_array = self.np.array(img) + + return img_array + except Exception as e: + self.P(f"Error decoding image: {e}", color='r') + return None + + def _extract_image_info(self, img_array): + """ + Extract comprehensive information from image array. + + Parameters + ---------- + img_array : np.ndarray + Image as numpy array + + Returns + ------- + dict + Dictionary with image properties + """ + if img_array is None or len(img_array.shape) < 2: + return { + 'error': 'Invalid image data', + 'valid': False + } + + # Extract basic dimensions + height, width = img_array.shape[:2] + channels = img_array.shape[2] if len(img_array.shape) > 2 else 1 + + # Calculate size info + total_pixels = height * width + size_mb = img_array.nbytes / (1024 * 1024) + + result = { + 'valid': True, + 'width': int(width), + 'height': int(height), + 'channels': int(channels), + 'total_pixels': int(total_pixels), + 'size_mb': round(size_mb, 3), + 'dtype': str(img_array.dtype), + 'shape': list(img_array.shape), + } + + # Add color information for RGB images + if channels >= 3: + result['color_info'] = { + 'mean_r': float(img_array[:, :, 0].mean()), + 'mean_g': float(img_array[:, :, 1].mean()), + 'mean_b': float(img_array[:, :, 2].mean()), + 'std_r': float(img_array[:, :, 0].std()), + 'std_g': float(img_array[:, :, 1].std()), + 'std_b': float(img_array[:, :, 2].std()), + } + + # Add quality assessment + result['quality_info'] = { + 'resolution_category': self._categorize_resolution(width, height), + 'aspect_ratio': round(width / height, 3) if height > 0 else 0, + 'is_square': abs(width - height) < 10, + } + + return result + + def _categorize_resolution(self, width, height): + """ + Categorize image resolution for quality assessment. + + Parameters + ---------- + width : int + Image width in pixels + height : int + Image height in pixels + + Returns + ------- + str + Resolution category + """ + total_pixels = width * height + + if total_pixels < 100000: # < 0.1 MP + return 'very_low' + elif total_pixels < 500000: # < 0.5 MP + return 'low' + elif total_pixels < 2000000: # < 2 MP + return 'medium' + elif total_pixels < 5000000: # < 5 MP + return 'high' + else: + return 'very_high' + + def _pre_process(self, inputs): + """ + Pre-process inputs: decode base64 images to numpy arrays. + + Parameters + ---------- + inputs : dict + Input dictionary with 'DATA' key containing list of base64 images + + Returns + ------- + list + List of decoded image arrays + """ + lst_inputs = inputs.get('DATA', []) + serving_params = inputs.get('SERVING_PARAMS', []) + + self.P(f"Pre-processing {len(lst_inputs)} input(s)", color='b') + + # DEBUG: Log what we received + for i, inp in enumerate(lst_inputs): + if isinstance(inp, dict): + self.P(f" Input #{i} keys: {list(inp.keys())}", color='y') + else: + self.P(f" Input #{i} type: {type(inp)}", color='y') + + preprocessed = [] + for i, inp in enumerate(lst_inputs): + # Get serving params for this specific input + params = serving_params[i] if i < len(serving_params) else {} + + # Decode the base64 image + img_array = self._decode_base64_image(inp) + + preprocessed.append({ + 'image': img_array, + 'params': params, + 'index': i, + }) + + return preprocessed + + def _predict(self, inputs): + """ + Main prediction: extract image information. + + In the future, this will call the actual AI model for cervical + cancer detection. + + Parameters + ---------- + inputs : list + List of preprocessed inputs (decoded images) + + Returns + ------- + list + List of analysis results + """ + self._processed_count += 1 + + results = [] + for inp_data in inputs: + img_array = inp_data['image'] + params = inp_data['params'] + idx = inp_data['index'] + + if img_array is None: + results.append({ + 'index': idx, + 'error': 'Failed to decode image', + 'valid': False + }) + continue + + # Extract image information + image_info = self._extract_image_info(img_array) + + # Add processing metadata + result = { + 'index': idx, + 'image_info': image_info, + 'processed_at': self.time(), + 'processor_version': __VER__, + 'model_name': 'cerviguard_image_analyzer', + 'iteration': self._processed_count, + } + + # TODO: Future enhancement - call AI model for cervical cancer detection + # if self.has_ai_model(): + # ai_prediction = self.run_ai_model(img_array) + # result['ai_analysis'] = ai_prediction + # result['risk_level'] = ai_prediction['risk_level'] + # result['confidence'] = ai_prediction['confidence'] + + results.append(result) + + return results + + def _post_process(self, preds): + """ + Post-process predictions: format for output. + + Parameters + ---------- + preds : list + List of prediction results + + Returns + ------- + list + Formatted results ready for return + """ + self.P(f"Post-processing {len(preds)} result(s)", color='b') + + formatted_results = [] + for pred in preds: + # Format the result for output + formatted = { + 'status': 'completed' if pred.get('image_info', {}).get('valid', False) else 'error', + 'data': pred, + } + + # Add error message if present + if 'error' in pred: + formatted['error'] = pred['error'] + + formatted_results.append(formatted) + + return formatted_results diff --git a/ver.py b/ver.py index 514c20dc..38a390b5 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.890' +__VER__ = '2.9.891' From 668cab5b05dc2a09fc877cd54331950d2b3f8cde Mon Sep 17 00:00:00 2001 From: Cristi Bleotiu <164478159+cristibleotiu@users.noreply.github.com> Date: Thu, 13 Nov 2025 21:45:19 +0200 Subject: [PATCH 02/11] feat: integration of both vllm and llama_cpp for inference (#309) * feat: integration of both vllm and llama_cpp for inference * chore: inc ver --- .devcontainer/Dockerfile | 19 +- Dockerfile_devnet | 4 +- Dockerfile_mainnet | 17 +- Dockerfile_testnet | 8 +- constants.py | 1 + .../container_apps/container_app_runner.py | 36 +- extensions/business/jeeves/jeeves_api.py | 7 + extensions/business/mixins/nlp_agent_mixin.py | 2 + extensions/business/nlp/vllm_agent.py | 646 ++++++++++++++++++ extensions/serving/ai_engines/stable.py | 12 + .../serving/base/base_doc_emb_serving.py | 8 +- extensions/serving/base/base_llm_serving.py | 29 +- .../default_inference/nlp/llama_cpp_base.py | 289 ++++++++ .../nlp/llama_cpp_llama_1b.py | 73 ++ .../nlp/llama_cpp_llama_3b.py | 73 ++ .../nlp/llama_cpp_llama_8b.py | 73 ++ .../default_inference/nlp/openai_server.py | 4 +- .../serving/mixins_llm/llm_model_mixin.py | 6 +- requirements.txt | 1 + ver.py | 2 +- 20 files changed, 1269 insertions(+), 41 deletions(-) create mode 100644 extensions/business/nlp/vllm_agent.py create mode 100644 extensions/serving/default_inference/nlp/llama_cpp_base.py create mode 100644 extensions/serving/default_inference/nlp/llama_cpp_llama_1b.py create mode 100644 extensions/serving/default_inference/nlp/llama_cpp_llama_3b.py create mode 100644 extensions/serving/default_inference/nlp/llama_cpp_llama_8b.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 57aacf2b..20e35f9d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,18 @@ -FROM ratio1/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-d +#FROM ratio1/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-d +FROM ratio1/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-dnctk # Install IPFS -RUN apt-get update && apt-get install -y wget && apt-get install -y tar +# The line below was creating issues due to flaky external repos +# RUN apt-get update && apt-get install -y wget && apt-get install -y tar +# Install tools needed for the next steps without hitting flaky external repos +RUN set -eux; \ + # disable NodeSource (if present) so apt won't read it + rm -f /etc/apt/sources.list.d/nodesource*.list /etc/apt/sources.list.d/nodesource*.sources || true; \ + apt-get update -o Acquire::Retries=3; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget tar ca-certificates \ + ninja-build; \ + rm -rf /var/lib/apt/lists/* RUN wget https://dist.ipfs.tech/kubo/v0.32.1/kubo_v0.32.1_linux-amd64.tar.gz && \ tar -xvzf kubo_v0.32.1_linux-amd64.tar.gz && \ @@ -32,9 +43,11 @@ RUN set -eux; \ # COPY ./cmds /usr/local/bin/ # RUN chmod +x /usr/local/bin/* +RUN npm install -g npm@latest @openai/codex WORKDIR /edge_node -COPY . . +#COPY . . +COPY requirements.txt . # RUN rm -rf /edge_node/cmds # set a generic env variable diff --git a/Dockerfile_devnet b/Dockerfile_devnet index c782cbe0..69ed89f1 100644 --- a/Dockerfile_devnet +++ b/Dockerfile_devnet @@ -10,7 +10,9 @@ RUN set -eux; \ # disable NodeSource (if present) so apt won't read it rm -f /etc/apt/sources.list.d/nodesource*.list /etc/apt/sources.list.d/nodesource*.sources || true; \ apt-get update -o Acquire::Retries=3; \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends wget tar ca-certificates; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget tar ca-certificates \ + ninja-build; \ rm -rf /var/lib/apt/lists/* RUN wget https://dist.ipfs.tech/kubo/v0.35.0/kubo_v0.35.0_linux-amd64.tar.gz && \ diff --git a/Dockerfile_mainnet b/Dockerfile_mainnet index f07dbfb8..5a5fb190 100644 --- a/Dockerfile_mainnet +++ b/Dockerfile_mainnet @@ -1,9 +1,20 @@ # -d for dind -FROM aidamian/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-d +#FROM aidamian/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-d +FROM ratio1/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-dnctk # Install IPFS -RUN apt-get update && apt-get install -y wget && apt-get install -y tar - +# The line below was creating issues due to flaky external repos +# RUN apt-get update && apt-get install -y wget && apt-get install -y tar +# Install tools needed for the next steps without hitting flaky external repos +RUN set -eux; \ + # disable NodeSource (if present) so apt won't read it + rm -f /etc/apt/sources.list.d/nodesource*.list /etc/apt/sources.list.d/nodesource*.sources || true; \ + apt-get update -o Acquire::Retries=3; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget tar ca-certificates \ + ninja-build; \ + rm -rf /var/lib/apt/lists/* + RUN wget https://dist.ipfs.tech/kubo/v0.35.0/kubo_v0.35.0_linux-amd64.tar.gz && \ tar -xvzf kubo_v0.35.0_linux-amd64.tar.gz && \ cd kubo && \ diff --git a/Dockerfile_testnet b/Dockerfile_testnet index 26591723..370493d8 100644 --- a/Dockerfile_testnet +++ b/Dockerfile_testnet @@ -1,6 +1,6 @@ # -d for dind -FROM aidamian/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-d -#FROM ratio1/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-dnctk +#FROM aidamian/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-d +FROM ratio1/base_edge_node:x86_64-py3.10.12-th2.3.1.cu121-tr4.43.3-dnctk # Install IPFS # The line below was creating issues due to flaky external repos @@ -10,7 +10,9 @@ RUN set -eux; \ # disable NodeSource (if present) so apt won't read it rm -f /etc/apt/sources.list.d/nodesource*.list /etc/apt/sources.list.d/nodesource*.sources || true; \ apt-get update -o Acquire::Retries=3; \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends wget tar ca-certificates; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget tar ca-certificates \ + ninja-build; \ rm -rf /var/lib/apt/lists/* RUN wget https://dist.ipfs.tech/kubo/v0.35.0/kubo_v0.35.0_linux-amd64.tar.gz && \ diff --git a/constants.py b/constants.py index c9703470..bae8e06e 100644 --- a/constants.py +++ b/constants.py @@ -165,6 +165,7 @@ class JeevesCt: JEEVES_AGENT_SIGNATURES = [ "DOC_EMBEDDING_AGENT", "LLM_AGENT", + "VLLM_AGENT", ] JEEVES_PLUGIN_SIGNATURES = JEEVES_API_SIGNATURES + JEEVES_AGENT_SIGNATURES diff --git a/extensions/business/container_apps/container_app_runner.py b/extensions/business/container_apps/container_app_runner.py index 26f8c930..d9fa8aab 100644 --- a/extensions/business/container_apps/container_app_runner.py +++ b/extensions/business/container_apps/container_app_runner.py @@ -128,6 +128,7 @@ "memory": "512m", # e.g. "512m" for 512MB, "ports": [] # dict of host_port: container_port mappings (e.g. {8080: 8081}) or list of container ports (e.g. [8080, 9000]) }, + "USE_CUDA": False, # If True, will use nvidia runtime for GPU support "RESTART_POLICY": "always", # "always" will restart the container if it stops "IMAGE_PULL_POLICY": "always", # "always" will always pull the image "AUTOUPDATE" : True, # If True, will check for image updates and pull them if available @@ -799,17 +800,18 @@ def read_all_extra_tunnel_logs(self): def start_container(self): """Start the Docker container.""" - self.P(f"Launching container with image '{self.cfg_image}'...") - - self.P(f"Container data:") - self.P(f" Image: {self.cfg_image}") - self.P(f" Ports: {self.json_dumps(self.inverted_ports_mapping) if self.inverted_ports_mapping else 'None'}") - self.P(f" Env: {self.json_dumps(self.env) if self.env else 'None'}") - self.P(f" Volumes: {self.json_dumps(self.volumes) if self.volumes else 'None'}") - self.P(f" Resources: {self.json_dumps(self.cfg_container_resources) if self.cfg_container_resources else 'None'}") - self.P(f" Restart policy: {self.cfg_restart_policy}") - self.P(f" Pull policy: {self.cfg_image_pull_policy}") - self.P(f" Start command: {self._start_command if self._start_command else 'Image default'}") + log_str = f"Launching container with image '{self.cfg_image}'..." + + log_str += f"Container data:" + log_str += f" Image: {self.cfg_image}" + log_str += f" Ports: {self.json_dumps(self.inverted_ports_mapping) if self.inverted_ports_mapping else 'None'}" + log_str += f" Env: {self.json_dumps(self.env) if self.env else 'None'}" + log_str += f" Volumes: {self.json_dumps(self.volumes) if self.volumes else 'None'}" + log_str += f" Resources: {self.json_dumps(self.cfg_container_resources) if self.cfg_container_resources else 'None'}" + log_str += f" Restart policy: {self.cfg_restart_policy}" + log_str += f" Pull policy: {self.cfg_image_pull_policy}" + log_str += f" Start command: {self._start_command if self._start_command else 'Image default'}" + self.P(log_str) try: run_kwargs = dict( @@ -822,6 +824,18 @@ def start_container(self): if self._start_command: run_kwargs['command'] = self._start_command + if self.cfg_use_cuda: + gpus_info = self.log.gpu_info() + if len(gpus_info) > 0: + run_kwargs['runtime'] = 'nvidia' + self.P(f"USE_CUDA is True and NVIDIA GPUs found, starting container with GPU support") + else: + self.P("Warning! USE_CUDA is True but no NVIDIA GPUs found, starting container without GPU support") + # endif available GPUs + else: + self.P(f"Starting container without GPU support") + # endif cfg_use_cuda + self.container = self.docker_client.containers.run( self.cfg_image, **run_kwargs, diff --git a/extensions/business/jeeves/jeeves_api.py b/extensions/business/jeeves/jeeves_api.py index 63d3757b..bd27cb22 100644 --- a/extensions/business/jeeves/jeeves_api.py +++ b/extensions/business/jeeves/jeeves_api.py @@ -2707,6 +2707,13 @@ def get_last_user_message(self, user_messages: list[dict]): # endfor return None + @_NetworkProcessorMixin.payload_handler(signature="VLLM_AGENT") + def handle_payload_vllm_agent(self, data): + return self.handle_payload_helper( + data=data, + agent_type="LLM", + ) + @_NetworkProcessorMixin.payload_handler(signature="LLM_AGENT") def handle_payload_llm_agent(self, data): return self.handle_payload_helper( diff --git a/extensions/business/mixins/nlp_agent_mixin.py b/extensions/business/mixins/nlp_agent_mixin.py index ff38fc86..5d63f10a 100644 --- a/extensions/business/mixins/nlp_agent_mixin.py +++ b/extensions/business/mixins/nlp_agent_mixin.py @@ -34,6 +34,8 @@ def inference_to_response(self, inference, model_name): def handle_inferences(self, inferences, data=None): if not isinstance(inferences, list): return + if len(inferences) > 0 and not isinstance(inferences[0], dict): + return model_name = inferences[0].get('MODEL_NAME', None) if len(inferences) > 0 else None cnt_initial_inferences = len(inferences) inferences, valid_idxs = self.filter_valid_inferences(inferences, return_idxs=True) diff --git a/extensions/business/nlp/vllm_agent.py b/extensions/business/nlp/vllm_agent.py new file mode 100644 index 00000000..e599cae7 --- /dev/null +++ b/extensions/business/nlp/vllm_agent.py @@ -0,0 +1,646 @@ +# from naeural_core.business.base.network_processor import NetworkProcessorPlugin as BasePlugin +from naeural_core.business.base import BasePluginExecutor as BasePlugin +from extensions.business.mixins.nlp_agent_mixin import _NlpAgentMixin, NLP_AGENT_MIXIN_CONFIG + +from concurrent.futures.thread import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Dict, Any +from concurrent.futures import Future + +import psutil + + +__VER__ = '0.1.0.0' + +_CONFIG = { + # mandatory area + **BasePlugin.CONFIG, + **NLP_AGENT_MIXIN_CONFIG, + + "ALLOW_EMPTY_INPUTS": True, + "CONTAINER_CHECK_INTERVAL": 60, # seconds + + "REQUEST_TIMEOUT": 60, # seconds + + "MODEL_NAME": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "HUGGINGFACE_API_TOKEN": None, + "PROCESS_DELAY": 1, + + "USE_GPU": None, + + "THREAD_MAX_WORKERS": 4, + "DEFAULT_TEMPERATURE": 0.7, + "DEFAULT_TOP_P": 0.9, + "DEFAULT_MAX_TOKENS": 256, + "DEFAULT_REPETITION_PENALTY": 1.1, + + 'VALIDATION_RULES': { + **BasePlugin.CONFIG['VALIDATION_RULES'], + **NLP_AGENT_MIXIN_CONFIG['VALIDATION_RULES'], + }, +} + + +@dataclass +class _ReqEntry: + meta: Dict[str, Any] + future: Future + request_type: str + start_time: float = 0.0 + elapsed_time: float = 0.0 +# endclass + + +REQUESTS_MUTEX = "vllm_requests_mutex" +DEFAULT_REQUEST_TIMEOUT = 60 # seconds + + +class VllmAgentPlugin(BasePlugin, _NlpAgentMixin): + CONFIG = _CONFIG + + def on_init(self): + super(VllmAgentPlugin, self).on_init() + self._pending_requests: Dict[str, Dict] = {} + self._processing_requests: Dict[str, _ReqEntry] = {} + self._solved_requests: Dict[str, _ReqEntry] = {} + + self._executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="vllm_agent_executor") + self._session = self.requests.Session() + self.container_port = None + self.launched_container_config = None + self.launched_container_pipeline_name = None + self.maybe_persistence_load() + self.pipeline_name_to_cleanup = None + self.in_cleanup = self.defaultdict(bool) + self.delayed_iterations_for_cleanup = 0 + # Check if the container config existed and if so, check any changes + if self.container_port is not None: + current_configured_container_pipeline_config = self.compute_vllm_container_pipeline_config() + if self.launched_container_config != current_configured_container_pipeline_config: + self.P("Detected change in vLLM container pipeline configuration, resetting launched container config.") + self.Pd(f"Previous config: {self.json_dumps(self.launched_container_config, indent=2)}\nNew config: {self.json_dumps(current_configured_container_pipeline_config, indent=2)}") + if isinstance(self.launched_container_config, dict): + previous_pipeline_name = self.launched_container_config.get("NAME", "") + if previous_pipeline_name: + self.pipeline_name_to_cleanup = previous_pipeline_name + self.delayed_iterations_for_cleanup = 5 + self.P(f"Scheduling cleanup of previous vLLM container pipeline: {previous_pipeline_name}") + # endif previous pipeline name successfully retrieved + # endif existent container_pipeline_config + # endif config changed + # endif existent container_pipeline_config + self.last_container_running_ts = 0 + return + + def persistence_save(self): + self.cacheapi_save_pickle(obj={ + "container_port": self.container_port, + "launched_container_config": self.launched_container_config, + "launched_container_pipeline_name": self.launched_container_pipeline_name, + }) + return + + def maybe_persistence_load(self): + data = self.cacheapi_load_pickle() + if not isinstance(data, dict): + return + self.container_port = data.get("container_port", None) + self.launched_container_config = data.get("launched_container_config", None) + self.launched_container_pipeline_name = data.get("launched_container_pipeline_name", None) + return + + def get_base_container_url(self): + return f"http://0.0.0.0:{self.container_port}" + + def get_container_completion_url(self): + base_url = self.get_base_container_url() + return f"{base_url}/v1/chat/completions" + + def get_container_models_url(self): + base_url = self.get_base_container_url() + return f"{base_url}/v1/models" + + def get_timeout(self): + configured_timeout = self.cfg_request_timeout + if isinstance(configured_timeout, (int, float)) and configured_timeout: + return configured_timeout + return DEFAULT_REQUEST_TIMEOUT + + def get_hugging_face_api_token(self): + configured_token = self.cfg_huggingface_api_token + env_token = self.os_environ.get("EE_HF_TOKEN", None) + return configured_token or env_token or "" + + """VLLM CONTAINER MANAGEMENT METHODS""" + if True: + def __get_all_used_ports(self): + res = set() + for conn in psutil.net_connections(kind='all'): + # Local address is not always a tuple. + if not isinstance(conn.laddr, str): + res.add(conn.laddr.port) # Local port + # endfor + return sorted(res) + + def get_start_command(self, port: int, model_name: str, use_gpu: bool): + """ + Method to get the start command for the vLLM server container. + Parameters + ---------- + port : int + The port on which the vLLM server will listen. + model_name : str + The name of the model to load. + use_gpu : bool + Whether to use GPU or not. + Returns + ------- + command : str + The command string to start the vLLM server. + """ + base_command = f"--host 0.0.0.0 --port {port} --model {model_name}" + cpu_cmd_suffix = f"--dtype float16 --disable-frontend-multiprocessing --disable-async-output-proc" + gpu_cmd_suffix = f"--kv-cache-dtype fp8 --gpu-memory-utilization 0.75 --quantization bitsandbytes" + cmd_suffix = gpu_cmd_suffix if use_gpu else cpu_cmd_suffix + return f"{base_command} {cmd_suffix}" + + def compute_vllm_container_instance_config(self): + """ + Method for computing plugin instance configuration of a ContainerAppRunner + that will run the vLLM server for the current plugin instance. + Returns + ------- + res : dict + The configuration dictionary for the vLLM container instance. + """ + # TODO: design a framework/algorithm/templates to configure the container based on resources + # and or preferences of the user(e.g. use GPU or not) + res = { + "INSTANCE_ID": f"{self.get_instance_id()}", + } + + if self.container_port is None: + used_ports = self.__get_all_used_ports() + chosen_port = self.np.random.randint(16000, 32000) + while chosen_port in used_ports: + chosen_port = self.np.random.randint(16000, 32000) + # endwhile used_port + self.container_port = chosen_port + # endif container_port is None + use_gpu = self.cfg_use_gpu + if use_gpu is None: + gpu_info = self.log.gpu_info() + use_gpu = len(gpu_info) > 0 + # endif use_gpu is None + + res["PORT"] = self.container_port + # TODO: review this + res["CONTAINER_RESOURCES"] = { + "cpu": 2, + "gpu": 1 if use_gpu else 0, + "memory": "10g", + "ports": { + str(self.container_port): str(self.container_port), + } + } + res["USE_CUDA"] = use_gpu + res["IMAGE"] = "vllm/vllm-openai:latest" if use_gpu else "substratusai/vllm:main-cpu" + res["ENV"] = { + "HUGGING_FACE_HUB_TOKEN": self.get_hugging_face_api_token(), + } + if use_gpu: + res["ENV"]["NVIDIA_VISIBLE_DEVICES"] = "all" + res["ENV"]["VLLM_ATTENTION_BACKEND"] = "FLASHINFER" + # endif use_gpu + res["TUNNEL_ENGINE_ENABLED"] = False + res["CONTAINER_START_COMMAND"] = self.get_start_command( + port=self.container_port, + model_name=self.cfg_model_name, + use_gpu=use_gpu, + ) + return res + + def check_vllm_container_running(self, skip_logs=False): + """ + Method to check if the vLLM container is already running for this plugin instance. + Returns + ------- + is_running : bool + True if the vLLM container is running, False otherwise. + """ + if self.time() - self.last_container_running_ts < self.cfg_container_check_interval: + return True + if not skip_logs: + self.Pd(f"Checking if vLLM container is running at port: {self.container_port}") + is_running = False + if self.container_port is None: + return is_running + + health_err = "" + health_msg = "" + models_msg = "" + try: + res = self.requests.get( + f"{self.get_base_container_url()}/health", + timeout=3 + ) + if res.status_code == 200: + is_running = True + health_msg = f"vLLM container health endpoint response status code: {res.status_code}" + except Exception as e: + health_err = f"vLLM container health endpoint check failed: {str(e)}" + + try: + res = self.get_models_request( + request_id="vllm_container_health_check", + timeout=3 + ) + models_msg = f"vLLM container models endpoint response: {self.json_dumps(res, indent=2)}" + + is_running = res.get("ok", False) + except Exception as e: + err_log = f"vLLM container models endpoint check failed: {str(e)}" + if health_err: + err_log = f"{health_err}; {err_log}" + health_err = err_log + # endtry + if not skip_logs: + if health_err: + self.Pd(health_err) + if not is_running: + if health_msg: + self.Pd(health_msg) + if models_msg: + self.Pd(models_msg) + # endif not is_running + # endif skip_logs + if is_running: + self.last_container_running_ts = self.time() + return is_running + + def check_vllm_container_pipeline_started(self): + return self.launched_container_config is not None + + def compute_vllm_container_pipeline_config(self): + car_config = self.compute_vllm_container_instance_config() + car_pipeline_config = { + "NAME": f"{self.get_stream_id()}__vllm", + "PLUGINS": [ + { + "INSTANCES": [ + car_config + ], + "SIGNATURE": "CONTAINER_APP_RUNNER" + } + ], + "TYPE": "VOID" + } + return car_pipeline_config + + def maybe_start_vllm_container(self): + if self.check_vllm_container_running(skip_logs=True): + return + if self.check_vllm_container_pipeline_started(): + return + car_pipeline_config = self.compute_vllm_container_pipeline_config() + self.Pd(f"Starting vLLM container with config: {self.json_dumps(car_pipeline_config, indent=2)}") + self.cmdapi_start_pipeline(self.deepcopy(car_pipeline_config)) + self.launched_container_config = car_pipeline_config + self.launched_container_pipeline_name = car_pipeline_config.get("NAME", None) + self.persistence_save() + return + + def maybe_clean_old_container_pipeline(self, pipeline_name: str = None): + deletion_started = False + pipeline_name = pipeline_name or self.pipeline_name_to_cleanup + if pipeline_name is None: + return deletion_started + if self.in_cleanup[pipeline_name]: + return deletion_started + current_node_pipeline = self.node_pipelines + current_pipeline_names = [p["NAME"] for p in current_node_pipeline] + if pipeline_name not in current_pipeline_names: + self.P(f"vLLM container pipeline: {pipeline_name} not found among current pipelines, assuming already deleted.") + return deletion_started + self.P(f"Stopping vLLM container pipeline: {pipeline_name}") + self.cmdapi_stop_pipeline( + node_address=None, + name=pipeline_name + ) + self.in_cleanup[pipeline_name] = True + launched_pipeline_name = self.launched_container_pipeline_name + extracted_pipeline_name = (self.launched_container_config or {}).get("NAME", None) + launched_pipeline_name = launched_pipeline_name or extracted_pipeline_name + if pipeline_name == launched_pipeline_name: + self.launched_container_config = None + self.launched_container_pipeline_name = None + self.container_port = None + self.persistence_save() + # endif launched container + deletion_started = True + return deletion_started + """END VLLM CONTAINER MANAGEMENT METHODS""" + + """REQUEST HANDLING METHODS""" + if True: + # TODO: maybe validate signature, model name or other aspects of payload + def check_relevant_data(self, data): + return True + + def check_relevant_request_type(self, request_type): + return True + + def extract_and_register_request(self, data): + added = False + self.Pd(f"Extracting and registering request from data: {self.json_dumps(data, indent=2)}") + if not self.check_relevant_data(data): + return added + + jeeves_content = data.get("JEEVES_CONTENT") + if not isinstance(jeeves_content, dict): + self.P(f"Invalid JEEVES_CONTENT type in data: expected dict, got {type(jeeves_content)}") + return added + + jeeves_content = { + k.upper() if isinstance(k, str) else k: v + for k, v in jeeves_content.items() + } + request_id = jeeves_content.get("REQUEST_ID", None) + if request_id is None: + self.P("Missing REQUEST_ID in JEEVES_CONTENT.") + return added + if not isinstance(request_id, str): + self.P(f"Invalid REQUEST_ID in JEEVES_CONTENT: expected str, got {type(request_id)}") + return added + # Maybe not mandatory? + request_id = request_id.strip() + request_type = jeeves_content.get("REQUEST_TYPE", "unknown") + if not self.check_relevant_request_type(request_type): + self.P(f"Irrelevant REQUEST_TYPE: {request_type}, skipping.") + return added + request_messages = jeeves_content.get("MESSAGES", []) + if not isinstance(request_messages, list) or len(request_messages) == 0: + self.P(f"Invalid or empty MESSAGES in JEEVES_CONTENT for request ID {request_id}. MESSAGES must be a non-empty list.") + return added + + request_temperature = jeeves_content.get("TEMPERATURE", self.cfg_default_temperature) + request_top_p = jeeves_content.get("TOP_P", self.cfg_default_top_p) + request_max_tokens = jeeves_content.get("MAX_TOKENS", self.cfg_default_max_tokens) + request_repetition_penalty = jeeves_content.get("REPETITION_PENALTY", self.cfg_default_repetition_penalty) + request_seed = jeeves_content.get("SEED", None) + # TODO: add tools support + + request_meta = { + "REQUEST_ID": request_id, + "REQUEST_TYPE": request_type, + "MESSAGES": request_messages, + "TEMPERATURE": request_temperature, + "TOP_P": request_top_p, + "MAX_TOKENS": request_max_tokens, + "REPETITION_PENALTY": request_repetition_penalty, + "SEED": request_seed, + } + self._pending_requests[request_id] = request_meta + added = True + return added + + def get_meta_from_request_data(self, request_data: Dict) -> Dict: + request_id = request_data["REQUEST_ID"] + request_type = request_data["REQUEST_TYPE"] + request_messages = request_data["MESSAGES"] + request_temperature = request_data["TEMPERATURE"] + request_top_p = request_data["TOP_P"] + request_max_tokens = request_data["MAX_TOKENS"] + request_repetition_penalty = request_data["REPETITION_PENALTY"] + request_seed = request_data["SEED"] + + payload = { + "model": self.cfg_model_name, + "messages": request_messages, + "stream": False, + } + additionals = { + "temperature": request_temperature, + "top_p": request_top_p, + "max_tokens": request_max_tokens, + "repetition_penalty": request_repetition_penalty, + "seed": request_seed, + } + normalized_additionals = { + k: v for k, v in additionals.items() if v is not None + } + payload.update(normalized_additionals) + + return { + "payload": payload, + } + + def _run_request(self, request_id: str, request_meta: Dict): + headers = {"Content-Type": "application/json"} + headers["X-Request-Id"] = request_id # echoed by vLLM if server flag enabled + + resp = self._session.post( + self.get_container_completion_url(), + headers=headers, + data=self.json_dumps(request_meta["payload"]), + timeout=self.get_timeout() + ) + resp.raise_for_status() + data = resp.json() + + # OpenAI-compatible shape: choices[0].message.content + content = data.get("choices", [{}])[0].get("message", {}).get("content") + return { + "request_id": request_id, + "ok": True, + "content": content, + "raw": data, + } + + def get_models_request(self, request_id: str, timeout: int = 5): + headers = {"Content-Type": "application/json"} + resp = self._session.get( + self.get_container_models_url(), + headers=headers, + timeout=timeout + ) + resp.raise_for_status() + data = resp.json() + return { + "request_id": request_id, + "ok": resp.status_code == 200, + "models": data.get("data", []), + "raw": data, + } + + def start_request(self, request_id: str, request_data: Dict, request_type: str): + self.P(f"Starting request ID: {request_id} with data: {self.json_dumps(request_data, indent=2)}") + start_time = self.time() + if request_type == "chat.completions": + request_meta = self.get_meta_from_request_data(request_data) + req_future = self._executor.submit( + self._run_request, + request_id, + request_meta, + ) + elif request_type == "models": + req_future = self._executor.submit( + self.get_models_request, + request_id, + ) + else: + self.P(f"Unknown request type: {request_type} for request ID: {request_id}") + return None + # endif request_type + req_entry = _ReqEntry( + meta=request_data, + future=req_future, + request_type=request_type, + start_time=start_time, + ) + self._processing_requests[request_id] = req_entry + return req_entry + + def maybe_start_pending_requests(self): + removed_ids = [] + for request_id, request_data in self._pending_requests.items(): + self.start_request( + request_id=request_id, + request_data=request_data, + request_type="chat.completions", + ) + removed_ids.append(request_id) + # endfor pending requests + for rid in removed_ids: + self._pending_requests.pop(rid, None) + # endfor removed_ids + return + + def check_request_finished(self, req_entry: _ReqEntry) -> bool: + if req_entry.future.done(): + req_entry.elapsed_time = self.time() - req_entry.start_time + return bool(req_entry.future.done()) + + def extract_request_result(self, request_id: str, req_entry: _ReqEntry) -> Dict: + try: + res = req_entry.future.result() + res = { + "MODEL_NAME": self.cfg_model_name, + "REQUEST_ID": request_id, + "IS_VALID": True, + "text": res.get("content", None), + "RAW": res.get("raw", None), + "ELAPSED_TIME": req_entry.elapsed_time, + } + except Exception as e: + res = { + "MODEL_NAME": self.cfg_model_name, + "REQUEST_ID": request_id, + "IS_VALID": False, + "ERROR": str(e), + "ELAPSED_TIME": req_entry.elapsed_time, + } + return res + + def maybe_handle_finished_requests(self): + removed_ids = [] + inferences = [] + datas = [] + for request_id, req_entry in self._processing_requests.items(): + if not self.check_request_finished(req_entry): + continue + result = self.extract_request_result(request_id=request_id, req_entry=req_entry) + removed_ids.append(request_id) + self._solved_requests[request_id] = result + inferences.append(result) + datas.append(req_entry.meta) + # endfor processing requests + self.handle_inferences(inferences=inferences, data=datas) + for rid in removed_ids: + self._processing_requests.pop(rid, None) + # endfor removed_ids + return + + """END REQUEST HANDLING METHODS""" + + def on_config(self): + new_container_config = self.compute_vllm_container_pipeline_config() + if isinstance(self.launched_container_config, dict) and self.launched_container_config != new_container_config: + self.P("Detected change in vLLM container pipeline configuration, resetting launched container config.") + debug_log = f"Previous config: {self.json_dumps(self.launched_container_config, indent=2)}\n" + debug_log += f"New config: {self.json_dumps(new_container_config, indent=2)}" + self.Pd(debug_log) + previous_pipeline_name = self.launched_container_config.get("NAME", "") + if previous_pipeline_name: + self.pipeline_name_to_cleanup = previous_pipeline_name + self.delayed_iterations_for_cleanup = 5 + self.P(f"Scheduling cleanup of previous vLLM container pipeline: {previous_pipeline_name}") + # endif previous pipeline name successfully retrieved + # endif config changed + return + + def on_close(self): + # Clean up the launched vLLM container pipeline if any + if self.launched_container_pipeline_name is not None: + self.P(f"Cleaning up launched vLLM container pipeline: {self.launched_container_pipeline_name} on plugin shutdown.") + self.cmdapi_stop_pipeline( + node_address=None, + name=self.launched_container_pipeline_name + ) + self.launched_container_pipeline_name = None + self.launched_container_config = None + self.container_port = None + self.persistence_save() + # endif launched container pipeline + super(VllmAgentPlugin, self).on_close() + return + + def _process(self): + # 1. Check if previous container needs cleanup + if self.pipeline_name_to_cleanup: + if self.delayed_iterations_for_cleanup > 0: + in_cleanup = self.in_cleanup[self.pipeline_name_to_cleanup] + log_prefix = "In cleanup of " if in_cleanup else "Will delete " + log_msg = log_prefix + f"vLLM container pipeline: {self.pipeline_name_to_cleanup}" + log_msg += f"[{self.delayed_iterations_for_cleanup} iterations left]." + self.P(log_msg) + self.delayed_iterations_for_cleanup -= 1 + return + # endif delayed_iterations_for_cleanup + if self.maybe_clean_old_container_pipeline(): + self.delayed_iterations_for_cleanup = 5 + log_str = f"Started deletion of vLLM container pipeline: {self.pipeline_name_to_cleanup}" + log_str += f", delaying further processing for cleanup." + self.P(log_str) + # endif started pipeline deletion + if self.delayed_iterations_for_cleanup == 0: + self.pipeline_name_to_cleanup = None + return + # endif deletion scheduled + + # 2. Ensure vLLM container is running + self.maybe_start_vllm_container() + + # 3. Check if container is ready + if not self.check_vllm_container_running(): + sleep_period = 10 + self.P(f"vLLM container not running yet, will retry after {sleep_period} seconds...") + self.sleep(sleep_period) + return + + # 4. Process incoming data and add to pending requests + datas = self.dataapi_struct_datas() + if datas: + self.P(f"Received {self.json_dumps(datas)}") + for d_key, data in datas.items(): + self.extract_and_register_request(data) + # endfor data + + # 5. Start pending requests if any + self.maybe_start_pending_requests() + + # 6. Collect finished requests if any + self.maybe_handle_finished_requests() + return + + diff --git a/extensions/serving/ai_engines/stable.py b/extensions/serving/ai_engines/stable.py index 062084d7..1bef50fb 100644 --- a/extensions/serving/ai_engines/stable.py +++ b/extensions/serving/ai_engines/stable.py @@ -9,6 +9,18 @@ 'SERVING_PROCESS': 'llama_v31' } +AI_ENGINES['llama_cpp_small'] = { + 'SERVING_PROCESS': 'llama_cpp_llama_1b' +} + +AI_ENGINES['llama_cpp_medium'] = { + 'SERVING_PROCESS': 'llama_cpp_llama_3b' +} + +AI_ENGINES['llama_cpp_large'] = { + 'SERVING_PROCESS': 'llama_cpp_llama_8b' +} + AI_ENGINES['llm_reason'] = { 'SERVING_PROCESS': 'deepseek_r1_qwen_7b' } diff --git a/extensions/serving/base/base_doc_emb_serving.py b/extensions/serving/base/base_doc_emb_serving.py index f746398b..cc1581a6 100644 --- a/extensions/serving/base/base_doc_emb_serving.py +++ b/extensions/serving/base/base_doc_emb_serving.py @@ -213,7 +213,7 @@ def __context_identifier(self, context): return 'default' if context is None else f'context_{context}' def __db_cache_workspace(self, context): - return self.os_path.join(self.get_models_folder(), 'vectordb', self.cfg_model_name, context) + return self.os_path.join(self.get_models_folder(), 'vectordb', self.get_model_name(), context) def get_embedding_size(self): return DOC_EMBEDDING_SIZE @@ -541,7 +541,7 @@ def doc_embedding_validate_request_params(self, request_type, request_params): def get_additional_metadata(self): return { - 'MODEL_NAME': self.cfg_model_name, + 'MODEL_NAME': self.get_model_name(), 'EMBEDDING_SIZE': self.get_embedding_size(), 'MAX_SEGMENT_SIZE': MAX_SEGMENT_SIZE, 'CONTEXTS': [ @@ -783,7 +783,7 @@ def get_result_dict(self, request_id, docs=None, query=None, context_list=None, 'DOCS': docs, DocEmbCt.QUERY: query, 'CONTEXT_LIST': context_list, - 'MODEL_NAME': self.cfg_model_name, + 'MODEL_NAME': self.get_model_name(), DocEmbCt.ERROR_MESSAGE: error_message, **uppercase_kwargs } @@ -903,7 +903,7 @@ def _post_process(self, preds_batch): else: final_result.append({ "IS_VALID": False, - "MODEL_NAME": self.cfg_model_name, + "MODEL_NAME": self.get_model_name(), }) # endfor each total input return final_result diff --git a/extensions/serving/base/base_llm_serving.py b/extensions/serving/base/base_llm_serving.py index d0f79f80..e7a04f19 100644 --- a/extensions/serving/base/base_llm_serving.py +++ b/extensions/serving/base/base_llm_serving.py @@ -178,7 +178,7 @@ def __init__(self, **kwargs): self.model = None self.tokenizer = None self.device = None - self.__tps = self.deque(maxlen=128) + self._tps = self.deque(maxlen=128) self.padding_id = None self.processed_requests = set() super(BaseLlmServing, self).__init__(**kwargs) @@ -200,10 +200,12 @@ def hf_token(self): cfg_hf_token = self.cfg_hf_token return cfg_hf_token or env_hf_token + def get_model_name(self): + return self.cfg_model_name @property def hf_model(self): - return self.cfg_model_name + return self.get_model_name() @property @@ -235,7 +237,7 @@ def get_relevant_signatures(self): def get_local_path(self): models_cache = self.log.get_models_folder() - model_name = 'models/{}'.format(self.cfg_model_name) + model_name = 'models/{}'.format(self.get_model_name()) model_subfolder = model_name.replace('/', '--') path = self.os_path.join(models_cache, model_subfolder) if self.os_path.isdir(path): @@ -301,7 +303,7 @@ def _startup(self): self.P(" Found HuggingFace token '{}'".format(obfuscated)) #endif no token - if self.cfg_model_name is None: + if self.get_model_name() is None: msg = "No model name found. Please set it in config `MODEL_NAME`" raise ValueError(msg) #endif no model name @@ -385,16 +387,20 @@ def _warmup(self): def _setup_llm(self): return + def get_configured_device(self): + configured_device = self.cfg_default_device + configured_device = (configured_device or 'cpu').lower() + return configured_device def _setup_device(self): # check if GPU is available & log gpu_info = self.log.gpu_info() - if len(gpu_info) == 0: + configured_device = self.get_configured_device() + if len(gpu_info) == 0 or configured_device == 'cpu': self.device = th.device('cpu') else: # try default device # TODO: review method - configured_device = self.cfg_default_device if configured_device in ["cuda", "gpu"]: configured_device = "cuda:0" # endif configured_device @@ -422,6 +428,9 @@ def _setup_device(self): def _get_device_map(self): # TODO: Rewrite to fix for multiple GPUs device_map = "auto" + configured_device = self.get_configured_device() + if configured_device == 'cpu': + device_map = 'cpu' return device_map def check_relevant_input(self, input_dict: dict): @@ -834,13 +843,13 @@ def _predict(self, preprocessed_batch): np_batch_tokens = batch_tokens.cpu().numpy() self.th_utils.clear_cache() - # Calculate number of generated token per seconds and add it to __tps + # Calculate number of generated token per seconds and add it to _tps # in order to track inference performance. Generated padding is not # counted since it is an artefact of the batching strategy. batch_y_size = np_batch_tokens.shape[1] num_generated_toks = (yhat[:, batch_y_size:] != self.padding_id).astype(self.np.int32).sum().item() num_tps = num_generated_toks / elapsed - self.__tps.append(num_tps) + self._tps.append(num_tps) self.P("Model ran at {} tokens per second".format(num_tps)) # Decode each output in the batch, omitting the input tokens. @@ -932,7 +941,7 @@ def _post_process(self, preds_batch): # LlmCT.TPS : tps, **preds_batch[LlmCT.ADDITIONAL][i], # TODO: find a way to send the model metadata to the plugin, other than through the inferences. - 'MODEL_NAME': self.cfg_model_name + 'MODEL_NAME': self.get_model_name() } result.append(dct_result) # endfor each text @@ -947,7 +956,7 @@ def _post_process(self, preds_batch): "IS_VALID": False, LlmCT.TEXT: "", LlmCT.PRMP: "", - 'MODEL_NAME': self.cfg_model_name + 'MODEL_NAME': self.get_model_name() }) # endfor total inputs return final_result diff --git a/extensions/serving/default_inference/nlp/llama_cpp_base.py b/extensions/serving/default_inference/nlp/llama_cpp_base.py new file mode 100644 index 00000000..331d2a88 --- /dev/null +++ b/extensions/serving/default_inference/nlp/llama_cpp_base.py @@ -0,0 +1,289 @@ +from extensions.serving.base.base_llm_serving import BaseLlmServing as BaseServingProcess +from llama_cpp import Llama +from extensions.serving.mixins_llm.llm_utils import LlmCT + +__VER__ = "0.1.0" + + +MODEL_N_CTX_MIN_VALUE = 512 +MODEL_N_CTX_DEFAULT_VALUE = 4096 +MODEL_N_BATCH_DEFAULT_VALUE = 512 + + +_CONFIG = { + **BaseServingProcess.CONFIG, + + "DEFAULT_DEVICE" : "cpu", + + "SKIP_ERRORS" : True, + "DETERMINISTIC_MODE": False, # If True, will use deterministic algorithms in PyTorch + + # Possible values of None, 4, 8, 16, 32 + # where None is the default model config. + "MODEL_WEIGHTS_SIZE" : None, + + "MODEL_N_CTX": MODEL_N_CTX_DEFAULT_VALUE, + + "MODEL_NAME": None, + "MODEL_FILENAME": None, + + 'VALIDATION_RULES': { + **BaseServingProcess.CONFIG['VALIDATION_RULES'], + }, + +} + + +class LlamaCppBaseServingProcess(BaseServingProcess): + CONFIG = _CONFIG + + def _load_tokenizer(self): + # llama.cpp uses built-in tokenizer + return + + def _load_model(self): + model_id = self.get_model_name() + model_filename = self.cfg_model_filename + + n_ctx = self.cfg_model_n_ctx + if not isinstance(n_ctx, (int, float)): + n_ctx = MODEL_N_CTX_DEFAULT_VALUE + # endif not int/float + n_ctx = max(MODEL_N_CTX_MIN_VALUE, int(n_ctx)) + + self.P(f"Loading Llama_cpp model '{model_id}' from file '{model_filename}'") + + model_params = { + 'n_ctx': n_ctx, + 'seed': self.cfg_generation_seed, + 'n_batch': MODEL_N_BATCH_DEFAULT_VALUE, + } + + self.model = Llama.from_pretrained( + repo_id=model_id, + filename=model_filename, + cache_dir=self.cache_dir, + **model_params + ) + self.P("Model loaded successfully.") + return + + def maybe_add_context_to_messages( + self, + messages: list[dict], + context: list or str = None + ): + if not isinstance(messages, list): + self.maybe_exception("messages must be a list of {role, content} dicts") + # endif messages type check + if context is not None and isinstance(context, (list, str)) and len(context) > 0: + if isinstance(context, str): + context = [context] + # endif context is str + context = [c for c in context if isinstance(c, str) and len(c) > 0] + # endif non-empty chat + # endif context provided + valid_messages = all( + isinstance(m, dict) and LlmCT.ROLE_KEY in m and LlmCT.DATA_KEY in m + for m in messages + ) + if not valid_messages: + msg = f"Each message in `messages` must be a dict with `role` and `content` keys. Invalid messages:\n{messages}" + self.maybe_exception(msg) + # endif valid messages + if not isinstance(context, list) or len(context) == 0: + return messages + # endif empty context + res, last_user_message, system_message = [], None, None + for message in messages: + role = message.get(LlmCT.ROLE_KEY, None) + content = message.get(LlmCT.DATA_KEY, None) + if role is None or content is None: + msg = f"Each message in `messages` must have a `role` and `content`. Invalid message:\n{message}" + self.maybe_exception(msg) + # endif role/content check + if role == LlmCT.SYSTEM_ROLE: + system_message = message + elif role == LlmCT.REQUEST_ROLE: + if last_user_message is not None: + res.append(last_user_message) + # endif last user message + last_user_message = message + elif role == LlmCT.REPLY_ROLE: + # assistant reply, so a new user message should come after this + if last_user_message is not None: + res.append(last_user_message) + last_user_message = None + # endif last user message + res.append(message) + # endif role check + # endfor messages + res = ([system_message] + res) if system_message is not None else res + if last_user_message is not None: + last_user_message_text = self.add_context_to_request( + last_user_message[LlmCT.DATA_KEY], + context + ) + last_user_message[LlmCT.DATA_KEY] = last_user_message_text + res.append(last_user_message) + # endif last user message + return res + + def _pre_process(self, inputs): + lst_inputs = inputs.get('DATA', []) + self.P(f"[DEBUG_LLM]Received {len(lst_inputs)} inputs for processing") + + predict_kwargs_lst = [] + messages_lst = [] + additional_lst = [] + valid_conditions = [] + process_methods = [] + relevant_input_ids = [] + cnt_total_inputs = len(lst_inputs) + + for i, inp in enumerate(lst_inputs): + if self.check_relevant_input(inp): + relevant_input_ids.append(i) + else: + continue + + jeeves_content = inp.get("JEEVES_CONTENT") + jeeves_content = { + (k.upper() if isinstance(k, str) else k): v + for k, v in jeeves_content.items() + } + request_id = jeeves_content.get(LlmCT.REQUEST_ID, None) + messages = jeeves_content.get(LlmCT.MESSAGES, []) + temperature = jeeves_content.get(LlmCT.TEMPERATURE) or self.cfg_default_temperature + top_p = jeeves_content.get(LlmCT.TOP_P) or self.cfg_default_top_p + max_tokens = jeeves_content.get(LlmCT.MAX_TOKENS) or self.cfg_default_max_tokens + repetition_penalty = jeeves_content.get("REPETITION_PENALTY", self.cfg_repetition_penalty) + request_context = jeeves_content.get(LlmCT.CONTEXT, None) + valid_condition = jeeves_content.get(LlmCT.VALID_CONDITION, None) + process_method = jeeves_content.get(LlmCT.PROCESS_METHOD, None) + predict_kwargs = { + 'temperature': temperature, + 'top_p': top_p, + 'max_tokens': max_tokens, + 'repeat_penalty': repetition_penalty, + } + if not isinstance(messages, list): + msg = f"Each input must have a list of messages. Received {type(messages)}: {self.shorten_str(inp)}" + self.maybe_exception(msg) + # endif messages not list + processed_messages = self.maybe_add_context_to_messages( + messages=messages, + context=request_context + ) + messages_lst.append(processed_messages) + predict_kwargs_lst.append(predict_kwargs) + additional_lst.append({ + LlmCT.REQUEST_ID: request_id, + }) + valid_conditions.append(valid_condition) + process_methods.append(process_method) + # endfor lst_inputs + + return [ + predict_kwargs_lst, + messages_lst, + additional_lst, + valid_conditions, + process_methods, + relevant_input_ids, + cnt_total_inputs, + ] + + def _predict(self, preprocessed_batch): + [ + predict_kwargs_lst, + messages_lst, + additional_lst, + valid_conditions, + process_methods, + relevant_input_ids, + cnt_total_inputs, + ] = preprocessed_batch + + results = [ + # (idx, valid, process_method, reply) + (idx, valid_condition, process_methods[idx], None) + for idx, valid_condition in enumerate(valid_conditions) + ] + obj_for_inference = [ + # original index, current index + (idx, idx) for idx in range(len(valid_conditions)) + ] + conditions_satisfied = False if len(valid_conditions) > 0 else True + max_tries = 10 + tries = 0 + while not conditions_satisfied: + reply_lst = [] + t0 = self.time() + timings = [] + total_generated_tokens = 0 + for idx_orig, idx_curr in obj_for_inference: + messages = messages_lst[idx_orig] + predict_kwargs = predict_kwargs_lst[idx_orig] + t1 = self.time() + out = self.model.create_chat_completion( + messages=messages, + **predict_kwargs + ) + elapsed = self.time() - t1 + timings.append(elapsed) + reply = out["choices"][0]["message"]["content"] + num_tokens_generated = out["usage"]["completion_tokens"] + total_generated_tokens += num_tokens_generated + reply_lst.append(reply) + # endfor obj_for_inference + t_total = self.time() - t0 + curr_tps = total_generated_tokens / t_total if t_total > 0 else 0 + self._tps.append(curr_tps) + self.P(f"Model ran at {curr_tps:.3f} tokens per second") + + invalid_objects = [] + tries += 1 + for idx_orig, idx_curr in obj_for_inference: + valid_condition = results[idx_orig][1] + process_method = results[idx_orig][2] + current_text = reply_lst[idx_curr] + self.P(f"Checking condition for object {idx_orig}:\nvalid:`{valid_condition}`|process:`{process_method}`|text:\n{current_text}") + current_text = self.maybe_process_text(current_text, process_method) + self.P(f"Processed text:\n{current_text}") + valid_text = ( + len(current_text) > 0 + and ( + valid_condition is None + or self.check_condition(current_text, valid_condition) + ) + ) + current_condition_satisfied = valid_text or (tries >= max_tries) + if current_condition_satisfied: + # If the condition is satisfied, we can save the result + results[idx_orig] = (idx_orig, valid_condition, process_method, current_text) + else: + invalid_objects.append((idx_orig, len(invalid_objects))) + # endif current condition satisfied + # endfor obj_for_inference + + if len(invalid_objects) > 0 and tries < max_tries: + obj_for_inference = invalid_objects + else: + conditions_satisfied = True + # endwhile conditions_satisfied + + text_lst = [text for _, _, _, text in results] + dct_result = { + LlmCT.PRMP: messages_lst, + LlmCT.TEXT: text_lst, + LlmCT.ADDITIONAL: additional_lst, + "RELEVANT_IDS": relevant_input_ids, + "TOTAL_INPUTS": cnt_total_inputs + } + return dct_result + + def _post_process(self, preds_batch): + # This method can be missing here, but is present in case + # of future customizations. + return super(LlamaCppBaseServingProcess, self)._post_process(preds_batch) diff --git a/extensions/serving/default_inference/nlp/llama_cpp_llama_1b.py b/extensions/serving/default_inference/nlp/llama_cpp_llama_1b.py new file mode 100644 index 00000000..cd1fabb8 --- /dev/null +++ b/extensions/serving/default_inference/nlp/llama_cpp_llama_1b.py @@ -0,0 +1,73 @@ +""" +@misc{touvron2023llamaopenefficientfoundation, + title={LLaMA: Open and Efficient Foundation Language Models}, + author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample}, + year={2023}, + eprint={2302.13971}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2302.13971}, +} + +models: + meta-llama/Meta-Llama-3.1-8B + meta-llama/Meta-Llama-3.1-8B-Instruct + + meta-llama/Meta-Llama-3.1-70B + meta-llama/Meta-Llama-3.1-70B-Instruct + + + meta-llama/Meta-Llama-3.1-405B + meta-llama/Meta-Llama-3.1-405B-FP8 + meta-llama/Meta-Llama-3.1-405B-Instruct + meta-llama/Meta-Llama-3.1-405B-Instruct-FP8 + + +Testing: + A. Launch OnDemandTextInput with Explorer + B. Write custom command (see below) + + + +for llama3.1 in-filling: +```json +{ + "ACTION" : "PIPELINE_COMMAND", + "PAYLOAD" : { + "NAME": "llm_request", + "PIPELINE_COMMAND" : { + "STRUCT_DATA" : { + "request" : "What is the square root of 4?", + "history" : [ + { + "request" : "hello", + "response" : "Hello, how can I help you today?" + } + ], + "system_info" : "You are a funny university teacher. Your task is to help students with their learning journey." + } + } + } +} +``` +""" + +from extensions.serving.default_inference.nlp.llama_cpp_base import LlamaCppBaseServingProcess as BaseServingProcess + +__VER__ = '0.1.0.0' + +_CONFIG = { + **BaseServingProcess.CONFIG, + + "MODEL_NAME": "hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF", + "MODEL_FILENAME": "llama-3.2-1b-instruct-q4_k_m.gguf", + + 'VALIDATION_RULES': { + **BaseServingProcess.CONFIG['VALIDATION_RULES'], + }, + +} + + +class LlamaCppLlama1B(BaseServingProcess): + CONFIG = _CONFIG diff --git a/extensions/serving/default_inference/nlp/llama_cpp_llama_3b.py b/extensions/serving/default_inference/nlp/llama_cpp_llama_3b.py new file mode 100644 index 00000000..38c3dc05 --- /dev/null +++ b/extensions/serving/default_inference/nlp/llama_cpp_llama_3b.py @@ -0,0 +1,73 @@ +""" +@misc{touvron2023llamaopenefficientfoundation, + title={LLaMA: Open and Efficient Foundation Language Models}, + author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample}, + year={2023}, + eprint={2302.13971}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2302.13971}, +} + +models: + meta-llama/Meta-Llama-3.1-8B + meta-llama/Meta-Llama-3.1-8B-Instruct + + meta-llama/Meta-Llama-3.1-70B + meta-llama/Meta-Llama-3.1-70B-Instruct + + + meta-llama/Meta-Llama-3.1-405B + meta-llama/Meta-Llama-3.1-405B-FP8 + meta-llama/Meta-Llama-3.1-405B-Instruct + meta-llama/Meta-Llama-3.1-405B-Instruct-FP8 + + +Testing: + A. Launch OnDemandTextInput with Explorer + B. Write custom command (see below) + + + +for llama3.1 in-filling: +```json +{ + "ACTION" : "PIPELINE_COMMAND", + "PAYLOAD" : { + "NAME": "llm_request", + "PIPELINE_COMMAND" : { + "STRUCT_DATA" : { + "request" : "What is the square root of 4?", + "history" : [ + { + "request" : "hello", + "response" : "Hello, how can I help you today?" + } + ], + "system_info" : "You are a funny university teacher. Your task is to help students with their learning journey." + } + } + } +} +``` +""" + +from extensions.serving.default_inference.nlp.llama_cpp_base import LlamaCppBaseServingProcess as BaseServingProcess + +__VER__ = '0.1.0.0' + +_CONFIG = { + **BaseServingProcess.CONFIG, + + "MODEL_NAME": "hugging-quants/Llama-3.2-3B-Instruct-Q4_K_M-GGUF", + "MODEL_FILENAME": "llama-3.2-3b-instruct-q4_k_m.gguf", + + 'VALIDATION_RULES': { + **BaseServingProcess.CONFIG['VALIDATION_RULES'], + }, + +} + + +class LlamaCppLlama3B(BaseServingProcess): + CONFIG = _CONFIG diff --git a/extensions/serving/default_inference/nlp/llama_cpp_llama_8b.py b/extensions/serving/default_inference/nlp/llama_cpp_llama_8b.py new file mode 100644 index 00000000..17eef0f7 --- /dev/null +++ b/extensions/serving/default_inference/nlp/llama_cpp_llama_8b.py @@ -0,0 +1,73 @@ +""" +@misc{touvron2023llamaopenefficientfoundation, + title={LLaMA: Open and Efficient Foundation Language Models}, + author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample}, + year={2023}, + eprint={2302.13971}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2302.13971}, +} + +models: + meta-llama/Meta-Llama-3.1-8B + meta-llama/Meta-Llama-3.1-8B-Instruct + + meta-llama/Meta-Llama-3.1-70B + meta-llama/Meta-Llama-3.1-70B-Instruct + + + meta-llama/Meta-Llama-3.1-405B + meta-llama/Meta-Llama-3.1-405B-FP8 + meta-llama/Meta-Llama-3.1-405B-Instruct + meta-llama/Meta-Llama-3.1-405B-Instruct-FP8 + + +Testing: + A. Launch OnDemandTextInput with Explorer + B. Write custom command (see below) + + + +for llama3.1 in-filling: +```json +{ + "ACTION" : "PIPELINE_COMMAND", + "PAYLOAD" : { + "NAME": "llm_request", + "PIPELINE_COMMAND" : { + "STRUCT_DATA" : { + "request" : "What is the square root of 4?", + "history" : [ + { + "request" : "hello", + "response" : "Hello, how can I help you today?" + } + ], + "system_info" : "You are a funny university teacher. Your task is to help students with their learning journey." + } + } + } +} +``` +""" + +from extensions.serving.default_inference.nlp.llama_cpp_base import LlamaCppBaseServingProcess as BaseServingProcess + +__VER__ = '0.1.0.0' + +_CONFIG = { + **BaseServingProcess.CONFIG, + + "MODEL_NAME": "joshnader/Meta-Llama-3.1-8B-Instruct-Q4_K_M-GGUF", + "MODEL_FILENAME": "meta-llama-3.1-8b-instruct-q4_k_m.gguf", + + 'VALIDATION_RULES': { + **BaseServingProcess.CONFIG['VALIDATION_RULES'], + }, + +} + + +class LlamaCppLlama8B(BaseServingProcess): + CONFIG = _CONFIG diff --git a/extensions/serving/default_inference/nlp/openai_server.py b/extensions/serving/default_inference/nlp/openai_server.py index aa5f1bf0..3d9a8493 100644 --- a/extensions/serving/default_inference/nlp/openai_server.py +++ b/extensions/serving/default_inference/nlp/openai_server.py @@ -118,7 +118,7 @@ def _predict(self, preprocessed_batch): ) response = self.model.chat.completions.create( - model=self.cfg_model_name, + model=self.get_model_name(), messages=messages, **predict_kwargs ) @@ -127,7 +127,7 @@ def _predict(self, preprocessed_batch): dct_result = { LlmCT.TEXT: response, **additional, - 'MODEL_NAME': self.cfg_model_name, + 'MODEL_NAME': self.get_model_name(), } results.append(dct_result) # endfor preprocessed_batch diff --git a/extensions/serving/mixins_llm/llm_model_mixin.py b/extensions/serving/mixins_llm/llm_model_mixin.py index 6fb11305..92626ca2 100644 --- a/extensions/serving/mixins_llm/llm_model_mixin.py +++ b/extensions/serving/mixins_llm/llm_model_mixin.py @@ -56,7 +56,7 @@ def str_device(dev): def _get_model_load_config(self): return self.log.get_model_load_config( - model_name=self.cfg_model_name, + model_name=self.get_model_name(), token=self.hf_token, has_gpu=self.has_gpu, weights_size=self.cfg_model_weights_size, @@ -93,7 +93,7 @@ def _load_tokenizer(self): # Load the tokenizer and output to log. cache_dir = self.cache_dir token = self.hf_token - model_id = self.cfg_model_name + model_id = self.get_model_name() self.P("Loading tokenizer for {} in '{}'...".format(model_id, cache_dir)) self.load_tokenizer(model_id, cache_dir, token) @@ -142,7 +142,7 @@ def _load_model(self): Will first set up the model loading configuration and then load the model """ - model_id = self.cfg_model_name + model_id = self.get_model_name() model_params, quantization_params = self._get_model_load_config() self.P("Loading {} with following parameters:\n{}\nQuantization params: {}".format( model_id, diff --git a/requirements.txt b/requirements.txt index 07743a42..bb6b438f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ sqlfluff pypdf python-docx pdfplumber +llama-cpp-python>=0.2.82 \ No newline at end of file diff --git a/ver.py b/ver.py index 38a390b5..c60fb580 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.891' +__VER__ = '2.9.892' From 6e2ce4cc1b88f0786ad370d54ce8e88b4a4abedb Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+vitalii-t12@users.noreply.github.com> Date: Fri, 14 Nov 2025 18:49:00 +0200 Subject: [PATCH 03/11] Cerviguard postponed request (#310) * fix: use postponed request for local serving API * fix: endpoints cleanup * fix: add analysis data in response * fix: add cerviguard to ai engines * fix: update ai_engine prop in config * fix: update alias * fix: rm alias * chore: inc version --- .../business/cerviguard/local_serving_api.py | 717 ++++++++---------- extensions/serving/ai_engines/stable.py | 4 + .../cerviguard/cerviguard_image_analyzer.py | 118 ++- ver.py | 2 +- 4 files changed, 434 insertions(+), 407 deletions(-) diff --git a/extensions/business/cerviguard/local_serving_api.py b/extensions/business/cerviguard/local_serving_api.py index dfd139c9..9cdf01f6 100644 --- a/extensions/business/cerviguard/local_serving_api.py +++ b/extensions/business/cerviguard/local_serving_api.py @@ -2,19 +2,30 @@ LOCAL_SERVING_API Plugin This plugin creates a FastAPI server for local-only access (localhost) that works with -a loopback data capture pipeline. It provides a simple API interface without token authentication, -suitable for internal/localhost-only services. +a loopback data capture pipeline. It uses the PostponedRequest pattern for async processing. + +Key Features: +- Loopback mode: Outputs return to DCT queue for processing +- PostponedRequest pattern: Server-side polling, no manual client polling +- No token authentication (localhost only) +- Designed for CerviGuard image analysis + +Available Endpoints: +- POST /predict - Submit image for analysis (returns result via PostponedRequest) +- GET /list_results - Get all processed image results +- GET /status - Get system status and statistics +- GET /health - Health check Example pipeline configuration: { - "NAME": "local_api_demo", + "NAME": "cerviguard_loopback", "TYPE": "Loopback", "PLUGINS": [ { "SIGNATURE": "LOCAL_SERVING_API", "INSTANCES": [ { - "INSTANCE_ID": "local_api_01" + "INSTANCE_ID": "cerviguard_api_01" } ] } @@ -23,44 +34,43 @@ """ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin +from naeural_core.utils.fastapi_utils import PostponedRequest __VER__ = '0.1.0' _CONFIG = { **FastApiWebAppPlugin.CONFIG, - # Mark this as a loopback plugin - outputs go back to the DCT queue instead of downstream + # Loopback mode - outputs go back to the DCT queue instead of downstream 'IS_LOOPBACK_PLUGIN': True, + # Server configuration 'PORT': 5082, - # Disable tunnel/ngrok since this is localhost only - 'TUNNEL_ENGINE_ENABLED': False, + 'TUNNEL_ENGINE_ENABLED': False, # Localhost only # API metadata - 'API_TITLE': 'Local Serving API', - 'API_SUMMARY': 'Local-only FastAPI server for internal services', - 'API_DESCRIPTION': 'A FastAPI server accessible only via localhost without token authentication', + 'API_TITLE': 'CerviGuard Local Serving API', + 'API_SUMMARY': 'Local image analysis API with PostponedRequest pattern', + 'API_DESCRIPTION': 'FastAPI server for cervical image analysis using loopback pipeline and PostponedRequest pattern', - # Response format - can be WRAPPED or RAW + # Response format 'RESPONSE_FORMAT': 'WRAPPED', - - # Enable request logging for debugging 'LOG_REQUESTS': True, - # Process delay + # Processing configuration 'PROCESS_DELAY': 0, - 'RESULT_CACHE_TTL': 300, + 'REQUEST_TIMEOUT': 240, # seconds - timeout for PostponedRequest polling - # AI Engine configuration for image analysis - 'AI_ENGINE': 'CERVIGUARD_IMAGE_ANALYZER', # Serving plugin to use + # AI Engine for image processing + 'AI_ENGINE': 'CERVIGUARD_IMAGE_ANALYZER', 'VALIDATION_RULES': { **FastApiWebAppPlugin.CONFIG['VALIDATION_RULES'], - 'RESULT_CACHE_TTL': { - 'DESCRIPTION': 'How long to keep results in cache (seconds)', + 'REQUEST_TIMEOUT': { + 'DESCRIPTION': 'Timeout for PostponedRequest polling (seconds)', 'TYPE': 'int', - 'MIN_VAL': 60, - 'MAX_VAL': 3600, + 'MIN_VAL': 30, + 'MAX_VAL': 600, }, }, } @@ -82,13 +92,12 @@ class LocalServingApiPlugin(FastApiWebAppPlugin): def on_init(self): super(LocalServingApiPlugin, self).on_init() - # Initialize instance variables - self._request_counter = 0 - self._data_buffer = [] - self._results_cache = {} - self._last_result_cleanup = self.time() - self.P("Local Serving API initialized - Loopback mode enabled", color='g') - self.P(f" Server accessible only on localhost (no tunnel)", color='g') + # Initialize request tracking + self.__requests = {} # Track active requests (PostponedRequest pattern) + self._data_buffer = [] # Simple activity log for monitoring + + self.P("Local Serving API initialized - Loopback + PostponedRequest mode", color='g') + self.P(f" Endpoints: /predict, /list_results, /status, /health", color='g') self.P(f" AI Engine: {self.cfg_ai_engine}", color='g') self.P(f" Loopback key: loopback_dct_{self._stream_id}", color='g') return @@ -103,255 +112,183 @@ def _get_payload_field(self, data: dict, key: str, default=None): return data[key_upper] return default - # ========== MOCKUP ENDPOINTS ========== - - @FastApiWebAppPlugin.endpoint(method="get") - def health(self): - """ - Health check endpoint - Returns server status and basic info - """ - return { - "status": "healthy", - "plugin": "LOCAL_SERVING_API", - "version": __VER__, - "stream_id": self._stream_id, - "instance_id": self.get_instance_id(), - "loopback_enabled": self.cfg_is_loopback_plugin, - "uptime_seconds": self.time() - self.start_time if hasattr(self, 'start_time') else 0, - } - - @FastApiWebAppPlugin.endpoint(method="get") - def status(self): - """ - Get current status and statistics - """ - return { - "request_count": self._request_counter, - "buffer_size": len(self._data_buffer), - "stream_id": self._stream_id, - "instance_id": self.get_instance_id(), - } + # ========== POSTPONED REQUEST METHODS ========== - @FastApiWebAppPlugin.endpoint(method="post") - def process_data(self, data: dict): + def register_predict_request(self, image_data: str, metadata: dict = None, request_type: str = 'prediction'): """ - Process arbitrary data and send it to the loopback queue + Register a new prediction request and add it to the loopback queue. Parameters ---------- - data : dict - The data to process + image_data : str + Base64 encoded image data + metadata : dict, optional + Additional metadata for the request + request_type : str + Type of prediction request (default: 'prediction') Returns ------- - dict - Processing result + str + The request ID """ - self._request_counter += 1 - request_id = self._request_counter + request_id = self.uuid() + start_time = self.time() - self.P(f"Processing data request #{request_id}") + # Register the request in the tracking dictionary + self.__requests[request_id] = { + 'request_id': request_id, + 'start_time': start_time, + 'last_request_time': start_time, + 'finished': False, + 'timeout': self.cfg_request_timeout, + 'type': request_type, + 'metadata': metadata or {}, + } - # Add to buffer + # Track in buffer for monitoring self._data_buffer.append({ - "request_id": request_id, - "data": data, - "timestamp": self.time() + 'request_id': request_id, + 'type': request_type, + 'submitted_at': start_time, + 'metadata': metadata or {} }) # Send to loopback queue via add_payload_by_fields - # This will automatically write to the loopback DCT queue because IS_LOOPBACK_PLUGIN=True + # Because IS_LOOPBACK_PLUGIN=True, this writes to loopback_dct_{stream_id} queue self.add_payload_by_fields( request_id=request_id, - input_data=data, - processed_at=self.time(), - source="local_serving_api" + image_data=image_data, + metadata=metadata or {}, + type=request_type, + submitted_at=start_time ) - return { - "request_id": request_id, - "status": "processed", - "message": "Data sent to loopback queue" - } - - @FastApiWebAppPlugin.endpoint(method="post") - def process_image(self, image_data: str, metadata: dict = None): - """ - Process image data (base64 encoded) and send to loopback + self.P(f"[Predict] Registered request {request_id} and added to loopback queue", color='g') - Parameters - ---------- - image_data : str - Base64 encoded image data - metadata : dict, optional - Additional metadata for the image + return request_id - Returns - ------- - dict - Processing result + def solve_postponed_predict_request(self, request_id: str): """ - self._request_counter += 1 - request_id = self._request_counter - - self.P(f"Processing image request #{request_id}") + Solver method for postponed prediction requests. - # In a real implementation, you would decode the base64 image - # For this mockup, we just log it - - payload = { - "request_id": request_id, - "image_size": len(image_data) if image_data else 0, - "metadata": metadata or {}, - "processed_at": self.time(), - "source": "local_serving_api_image" - } - - # Send to loopback queue - self.add_payload_by_fields(**payload) - - return { - "request_id": request_id, - "status": "image_processed", - "message": "Image data sent to loopback queue" - } - - # ========== CERVIGUARD WAR ENDPOINTS ========== - - @FastApiWebAppPlugin.endpoint(method="post") - def predict(self, image_data: str, metadata: dict = None): - """ - Simple /predict endpoint for image analysis - - Simplified endpoint that accepts an image and returns a request ID. - This is the main endpoint for the cerviguard flow: - 1. Receives base64 image - 2. Adds to loopback queue via add_payload_by_fields - 3. Serving plugin processes the image - 4. Results cached for polling + This method is called repeatedly by the FastAPI framework until the request + is finished or times out. Parameters ---------- - image_data : str - Base64 encoded image (supports data URLs) - metadata : dict, optional - Additional metadata + request_id : str + The request ID to check Returns ------- - dict - Request ID and status for polling + dict or PostponedRequest + Returns result dict if finished, or PostponedRequest to continue polling """ - # Generate unique request ID - request_id = self.uuid() - - self.P(f"[Predict] Received image, request_id: {request_id}", color='b') - - # Validate image data - if not image_data or len(image_data) < 100: + if request_id not in self.__requests: return { - "status": "error", - "error": "Invalid or missing image data", - "message": "Image data must be base64 encoded" + 'status': 'error', + 'error': 'Request ID not found', + 'request_id': request_id } - # Track request - self._data_buffer.append({ - "request_id": request_id, - "type": "prediction", - "submitted_at": self.time(), - "metadata": metadata or {} - }) - - # STEP 3: Send to loopback queue via add_payload_by_fields - # Because IS_LOOPBACK_PLUGIN=True, this writes to loopback_dct_{stream_id} queue - self.add_payload_by_fields( - request_id=request_id, - image_data=image_data, - metadata=metadata or {}, - type="prediction", - submitted_at=self.time() + request = self.__requests[request_id] + start_time = request['start_time'] + timeout = request['timeout'] + + # Check if request is finished + if request['finished']: + result = request.get('result', {}) + self.P(f"[Predict] Request {request_id} completed, returning result", color='g') + return result + + # Check if request has timed out + if self.time() - start_time > timeout: + error_result = { + 'status': 'error', + 'error': 'Request timed out', + 'request_id': request_id, + 'timeout': timeout + } + request['result'] = error_result + request['finished'] = True + self.P(f"[Predict] Request {request_id} timed out after {timeout}s", color='r') + return error_result + + # Request still processing - return PostponedRequest to continue polling + return self.create_postponed_request( + solver_method=self.solve_postponed_predict_request, + method_kwargs={'request_id': request_id} ) - self.P(f"[Predict] Image added to loopback queue: {request_id}", color='g') + # ========== API ENDPOINTS ========== + @FastApiWebAppPlugin.endpoint(method="get") + def health(self): + """ + Health check endpoint + Returns server status and basic info + """ return { - "status": "submitted", - "request_id": request_id, - "message": "Image queued for analysis", - "poll_endpoint": f"/get_result?request_id={request_id}" + "status": "healthy", + "plugin": "LOCAL_SERVING_API", + "version": __VER__, + "stream_id": self._stream_id, + "instance_id": self.get_instance_id(), + "loopback_enabled": self.cfg_is_loopback_plugin, + "uptime_seconds": self.time() - self.start_time if hasattr(self, 'start_time') else 0, } @FastApiWebAppPlugin.endpoint(method="get") - def get_result(self, request_id: str): + def status(self): """ - Get prediction result for /predict endpoint - - Poll this endpoint to retrieve the processing result. + Get current system status and statistics """ - if not request_id: - return { - "status": "error", - "error": "Missing request_id parameter" - } - - self.P(f"[Predict] Result requested for: {request_id}", color='b') - - result = self._results_cache.get(request_id) - - if result is None: - submitted = any( - item.get('request_id') == request_id - for item in self._data_buffer - ) - - if submitted: - return { - "status": "processing", - "request_id": request_id, - "message": "Image is still being processed, please poll again" - } - else: - return { - "status": "not_found", - "request_id": request_id, - "error": "Request ID not found" - } - - self.P(f"[Predict] Returning result for: {request_id}", color='g') - + pending = len([ + req_id for req_id, req_data in self.__requests.items() + if not req_data.get('finished', False) + ]) + completed = len([ + req_id for req_id, req_data in self.__requests.items() + if req_data.get('finished', False) + ]) return { - "status": "completed", - "request_id": request_id, - "result": result['result'] + "status": "online", + "service": "CerviGuard Image Analysis", + "version": __VER__, + "stream_id": self._stream_id, + "instance_id": self.get_instance_id(), + "total_requests": len(self.__requests), + "pending_requests": pending, + "completed_requests": completed, + "uptime_seconds": self.time() - self.start_time if hasattr(self, 'start_time') else 0, } @FastApiWebAppPlugin.endpoint(method="post") - def cerviguard_submit_image(self, image_data: str, metadata: dict = None): + def predict(self, image_data: str, metadata: dict = None): """ - CerviGuard WAR: Submit cervical image for analysis + Simple /predict endpoint for image analysis using PostponedRequest pattern - This endpoint is specifically designed for the CerviGuard WAR application. - It accepts a base64-encoded image, assigns a request ID, and queues it - for processing via the loopback mechanism. + This endpoint uses the PostponedRequest pattern to handle async processing: + 1. Receives base64 image + 2. Registers request and adds to loopback queue + 3. Returns PostponedRequest that framework polls automatically + 4. Serving plugin processes the image in the background + 5. When complete, result is returned to client Parameters ---------- image_data : str - Base64 encoded cervical image (supports data URLs) + Base64 encoded image (supports data URLs) metadata : dict, optional - Additional metadata (patient_id, capture_date, etc.) + Additional metadata Returns ------- - dict - Request ID and status for polling + dict or PostponedRequest + Either immediate error or PostponedRequest for async processing """ - # Generate unique request ID - request_id = self.uuid() - - self.P(f"[CerviGuard] Received image submission, request_id: {request_id}", color='b') + self.P(f"[Predict] Received image prediction request", color='b') # Validate image data if not image_data or len(image_data) < 100: @@ -361,173 +298,70 @@ def cerviguard_submit_image(self, image_data: str, metadata: dict = None): "message": "Image data must be base64 encoded" } - # Track request - self._data_buffer.append({ - "request_id": request_id, - "type": "cervical_analysis", - "submitted_at": self.time(), - "metadata": metadata or {} - }) - - # Send to loopback queue - self.add_payload_by_fields( - request_id=request_id, + # Register the request and add to loopback queue + request_id = self.register_predict_request( image_data=image_data, - metadata=metadata or {}, - type="cervical_analysis", - submitted_at=self.time() + metadata=metadata, + request_type='prediction' ) - self.P(f"[CerviGuard] Image queued for processing: {request_id}", color='g') - - return { - "status": "submitted", - "request_id": request_id, - "message": "Image queued for analysis", - "poll_endpoint": f"/cerviguard_get_result?request_id={request_id}" - } - - @FastApiWebAppPlugin.endpoint(method="get") - def cerviguard_get_result(self, request_id: str): - """ - CerviGuard WAR: Get image analysis result - - Poll this endpoint to retrieve the processing result for a submitted image. - """ - if not request_id: - return { - "status": "error", - "error": "Missing request_id parameter" - } + # Return PostponedRequest - framework will poll solve_postponed_predict_request() + return self.solve_postponed_predict_request(request_id=request_id) - self.P(f"[CerviGuard] Result requested for: {request_id}", color='b') - - result = self._results_cache.get(request_id) - - if result is None: - submitted = any( - item.get('request_id') == request_id - for item in self._data_buffer - ) - - if submitted: - return { - "status": "processing", - "request_id": request_id, - "message": "Image is still being processed, please poll again" - } - else: - return { - "status": "not_found", - "request_id": request_id, - "error": "Request ID not found" - } - - self.P(f"[CerviGuard] Returning result for: {request_id}", color='g') - - return { - "status": "completed", - "request_id": request_id, - "result": result['result'] - } - # - @FastApiWebAppPlugin.endpoint(method="get") - def cerviguard_status(self): - """ - CerviGuard WAR: Get system status - """ - pending = len([ - item for item in self._data_buffer - if item.get('request_id') not in self._results_cache - ]) - return { - "status": "online", - "service": "CerviGuard Image Analysis", - "version": __VER__, - "total_requests": self._request_counter, - "pending_requests": pending, - "uptime_seconds": self.time() - self.start_time if hasattr(self, 'start_time') else 0, - } @FastApiWebAppPlugin.endpoint(method="get") - def get_buffer(self, limit: int = 10): + def list_results(self, limit: int = 50, include_pending: bool = False): """ - Get recent data from the buffer + Get all processed image results Parameters ---------- limit : int - Maximum number of items to return (default: 10) + Maximum number of results to return (default: 50, max: 100) + include_pending : bool + Whether to include pending requests (default: False) Returns ------- dict - Buffer contents - """ - return { - "buffer_size": len(self._data_buffer), - "limit": limit, - "items": self._data_buffer[-limit:] if self._data_buffer else [] - } - - @FastApiWebAppPlugin.endpoint(method="post") - def clear_buffer(self): + List of all processed results with metadata """ - Clear the internal data buffer + # Limit validation + limit = min(max(1, limit), 100) + + results_list = [] + for req_id, req_data in self.__requests.items(): + is_finished = req_data.get('finished', False) + + # Skip pending if not requested + if not include_pending and not is_finished: + continue + + result_item = { + 'request_id': req_id, + 'type': req_data.get('type', 'unknown'), + 'status': 'completed' if is_finished else 'processing', + 'submitted_at': req_data.get('start_time'), + 'metadata': req_data.get('metadata', {}), + } - Returns - ------- - dict - Clear operation result - """ - prev_size = len(self._data_buffer) - self._data_buffer = [] + # Add result if finished + if is_finished: + result_item['result'] = req_data.get('result', {}) - return { - "status": "cleared", - "previous_size": prev_size, - "message": f"Cleared {prev_size} items from buffer" - } + results_list.append(result_item) - @FastApiWebAppPlugin.endpoint(method="post") - def batch_process(self, items: list): - """ - Process a batch of items + # Sort by submission time (most recent first) + results_list.sort(key=lambda x: x.get('submitted_at', 0), reverse=True) - Parameters - ---------- - items : list - List of items to process - - Returns - ------- - dict - Batch processing result - """ - if not isinstance(items, list): - return { - "error": "items must be a list", - "status": "failed" - } - - batch_id = self.uuid() - self._request_counter += len(items) - - # Process each item and send to loopback - for idx, item in enumerate(items): - self.add_payload_by_fields( - batch_id=batch_id, - item_index=idx, - item_data=item, - processed_at=self.time(), - source="local_serving_api_batch" - ) + # Apply limit + results_list = results_list[:limit] return { - "batch_id": batch_id, - "items_processed": len(items), - "status": "batch_completed", - "message": f"Processed {len(items)} items and sent to loopback queue" + "total_results": len(results_list), + "limit": limit, + "include_pending": include_pending, + "results": results_list } def process(self): @@ -536,9 +370,9 @@ def process(self): 1. Read struct_data from pipeline (contains image requests from loopback) 2. Read inferences from serving plugin (which has already processed them) 3. Match inferences to requests by index - 4. Cache results for retrieval via API + 4. Mark requests as finished for PostponedRequest polling """ - self._cleanup_result_cache() + self._cleanup_old_requests() self._maybe_trim_buffer() # Read struct_data from pipeline (raw payloads) @@ -568,7 +402,7 @@ def _process_loopback_payload(self, payload, inference): Process a single payload from loopback queue with its corresponding inference: 1. Extract request info from payload 2. Extract inference result from serving plugin - 3. Cache result for API retrieval + 3. Mark request as finished in __requests dict Parameters ---------- @@ -588,20 +422,25 @@ def _process_loopback_payload(self, payload, inference): self.P("Received payload without request_id, ignoring", color='y') return + # Check if this request is tracked + if request_id not in self.__requests: + self.P(f"Request {request_id} not found in tracked requests, ignoring", color='y') + return + if inference is None: self.P(f"No inference available for request {request_id}", color='y') - self._cache_error_result(request_id, 'No inference result available') + self._mark_request_error(request_id, 'No inference result available') return self.P(f"[CerviGuard] Processing inference for request {request_id}", color='b') try: # The serving plugin returns inferences in a specific format - # For CERVIGUARD_IMAGE_ANALYZER, it returns: {'status': 'completed', 'data': {...}} + # For cerviguard_analyzer, it returns: {'status': 'completed', 'data': {...}} if not isinstance(inference, dict): self.P(f"Unexpected inference format: {type(inference)}", color='r') - self._cache_error_result(request_id, 'Invalid inference result format') + self._mark_request_error(request_id, 'Invalid inference result format') return # Extract the inference data @@ -613,40 +452,125 @@ def _process_loopback_payload(self, payload, inference): if status == 'error': error_msg = inference_data.get('error', 'Unknown error') - self._cache_error_result(request_id, error_msg) + self._mark_request_error(request_id, error_msg) return - # Success - extract image info + # Success - extract image info and analysis image_info = inference_data.get('image_info', {}) + analysis = inference_data.get('analysis', {}) + + # Validate and provide safe defaults for analysis fields + validated_analysis = self._validate_analysis(analysis) + + # Include image_info in analysis + validated_analysis['image_info'] = image_info final_result = { 'status': 'completed', 'request_id': request_id, - 'image_info': image_info, + 'analysis': validated_analysis, 'processed_at': inference_data.get('processed_at', self.time()), 'processor_version': inference_data.get('processor_version', 'unknown'), 'metadata': metadata, } - # Cache the result - self._results_cache[request_id] = { - 'result': final_result, - 'stored_at': self.time(), - } + # Mark request as finished with result + self.__requests[request_id]['result'] = final_result + self.__requests[request_id]['finished'] = True - self.P(f"[CerviGuard] Cached result for request {request_id}", color='g') + self.P(f"[CerviGuard] Marked request {request_id} as finished", color='g') except Exception as e: self.P(f"Error processing request {request_id}: {e}", color='r') import traceback self.P(traceback.format_exc(), color='r') - self._cache_error_result(request_id, f'Processing error: {str(e)}') + self._mark_request_error(request_id, f'Processing error: {str(e)}') return - def _cache_error_result(self, request_id: str, error_message: str): - """Cache an error result for a request""" - self.P(f"[CerviGuard] Caching error for request {request_id}: {error_message}", color='r') + def _validate_analysis(self, analysis: dict) -> dict: + """ + Validate and sanitize analysis data, providing safe defaults. + + Parameters + ---------- + analysis : dict + Analysis data from serving plugin + + Returns + ------- + dict + Validated analysis with all required fields + """ + # Define safe defaults + safe_defaults = { + 'tz_type': 'Type 1', + 'lesion_assessment': 'none', + 'lesion_summary': 'Analysis unavailable', + 'risk_score': 0, + 'image_quality': 'unknown', + 'image_quality_sufficient': True + } + + if not isinstance(analysis, dict): + self.P("Analysis data is not a dict, using defaults", color='y') + return safe_defaults + + # Validate each field + validated = {} + + # Validate tz_type (must be "Type 1", "Type 2", or "Type 3") + tz_type = analysis.get('tz_type', safe_defaults['tz_type']) + if tz_type not in ['Type 1', 'Type 2', 'Type 3']: + self.P(f"Invalid tz_type: {tz_type}, using default", color='y') + tz_type = safe_defaults['tz_type'] + validated['tz_type'] = tz_type + + # Validate lesion_assessment (must be "none", "low", "moderate", or "high") + lesion_assessment = analysis.get('lesion_assessment', safe_defaults['lesion_assessment']) + if lesion_assessment not in ['none', 'low', 'moderate', 'high']: + self.P(f"Invalid lesion_assessment: {lesion_assessment}, using default", color='y') + lesion_assessment = safe_defaults['lesion_assessment'] + validated['lesion_assessment'] = lesion_assessment + + # Validate lesion_summary (must be string) + lesion_summary = analysis.get('lesion_summary', safe_defaults['lesion_summary']) + if not isinstance(lesion_summary, str): + self.P(f"Invalid lesion_summary type: {type(lesion_summary)}, using default", color='y') + lesion_summary = safe_defaults['lesion_summary'] + validated['lesion_summary'] = lesion_summary + + # Validate risk_score (must be int 0-100) + risk_score = analysis.get('risk_score', safe_defaults['risk_score']) + try: + risk_score = int(risk_score) + if risk_score < 0 or risk_score > 100: + self.P(f"risk_score out of range: {risk_score}, clamping to 0-100", color='y') + risk_score = max(0, min(100, risk_score)) + except (TypeError, ValueError): + self.P(f"Invalid risk_score: {risk_score}, using default", color='y') + risk_score = safe_defaults['risk_score'] + validated['risk_score'] = risk_score + + # Validate image_quality (must be string) + image_quality = analysis.get('image_quality', safe_defaults['image_quality']) + if not isinstance(image_quality, str): + self.P(f"Invalid image_quality type: {type(image_quality)}, using default", color='y') + image_quality = safe_defaults['image_quality'] + validated['image_quality'] = image_quality + + # Validate image_quality_sufficient (must be boolean) + image_quality_sufficient = analysis.get('image_quality_sufficient', safe_defaults['image_quality_sufficient']) + if not isinstance(image_quality_sufficient, bool): + self.P(f"Invalid image_quality_sufficient type: {type(image_quality_sufficient)}, using default", color='y') + image_quality_sufficient = safe_defaults['image_quality_sufficient'] + validated['image_quality_sufficient'] = image_quality_sufficient + + return validated + + def _mark_request_error(self, request_id: str, error_message: str): + """Mark a request as finished with an error""" + self.P(f"[CerviGuard] Marking request {request_id} as error: {error_message}", color='r') error_result = { 'status': 'error', @@ -655,27 +579,24 @@ def _cache_error_result(self, request_id: str, error_message: str): 'processed_at': self.time(), } - self._results_cache[request_id] = { - 'result': error_result, - 'stored_at': self.time(), - } + if request_id in self.__requests: + self.__requests[request_id]['result'] = error_result + self.__requests[request_id]['finished'] = True return - def _cleanup_result_cache(self): + def _cleanup_old_requests(self): + """Clean up old finished requests to prevent memory buildup""" now = self.time() - if now - self._last_result_cleanup < 60: - return - self._last_result_cleanup = now - - ttl = self.cfg_result_cache_ttl + # Clean up requests older than 1 hour + max_age = 3600 expired = [ - req_id for req_id, data in self._results_cache.items() - if now - data['stored_at'] > ttl + req_id for req_id, req_data in self.__requests.items() + if req_data.get('finished', False) and (now - req_data.get('start_time', now)) > max_age ] for req_id in expired: - del self._results_cache[req_id] + del self.__requests[req_id] if expired: - self.P(f"[CerviGuard] Cleaned {len(expired)} cached results", color='y') + self.P(f"[CerviGuard] Cleaned up {len(expired)} old finished requests", color='y') return def _maybe_trim_buffer(self): diff --git a/extensions/serving/ai_engines/stable.py b/extensions/serving/ai_engines/stable.py index 1bef50fb..e1f51ff0 100644 --- a/extensions/serving/ai_engines/stable.py +++ b/extensions/serving/ai_engines/stable.py @@ -49,3 +49,7 @@ 'SERVING_PROCESS': 'mxbai_embed' } +# AI_ENGINES['cerviguard_analyzer'] = { +# 'SERVING_PROCESS': 'cerviguard_image_analyzer' +# } + diff --git a/extensions/serving/cerviguard/cerviguard_image_analyzer.py b/extensions/serving/cerviguard/cerviguard_image_analyzer.py index 28204143..10fa4ece 100644 --- a/extensions/serving/cerviguard/cerviguard_image_analyzer.py +++ b/extensions/serving/cerviguard/cerviguard_image_analyzer.py @@ -19,7 +19,7 @@ "INSTANCES": [ { "INSTANCE_ID": "cerviguard_01", - "AI_ENGINE": "CERVIGUARD_IMAGE_ANALYZER" + "AI_ENGINE": "cerviguard_analyzer" } ] } @@ -204,6 +204,111 @@ def _categorize_resolution(self, width, height): else: return 'very_high' + def _generate_cervical_analysis(self, img_array, image_info): + """ + Generate cervical screening analysis results. + + This is a mock implementation that generates plausible analysis based on + image characteristics. In production, this would be replaced with actual + ML model inference for cervical cancer detection. + + Parameters + ---------- + img_array : np.ndarray + Image as numpy array + image_info : dict + Extracted image information + + Returns + ------- + dict + Analysis results with tz_type, lesion_assessment, lesion_summary, and risk_score + """ + # Mock implementation - generates deterministic results based on image characteristics + # In production, this would call an actual ML model + + if img_array is None or not image_info.get('valid', False): + return { + 'tz_type': 'Type 1', + 'lesion_assessment': 'none', + 'lesion_summary': 'Image quality insufficient for analysis', + 'risk_score': 0 + } + + # Use image characteristics to generate mock analysis + # In production, this would be replaced with actual model predictions + width = image_info.get('width', 0) + height = image_info.get('height', 0) + channels = image_info.get('channels', 3) + + # Generate mock TZ type (Type 1, Type 2, Type 3) + # Using image dimensions as seed for deterministic results + tz_seed = (width + height) % 3 + tz_types = ['Type 1', 'Type 2', 'Type 3'] + tz_type = tz_types[tz_seed] + + # Generate mock lesion assessment (none, low, moderate, high) + # Using color information if available + if 'color_info' in image_info: + mean_intensity = ( + image_info['color_info']['mean_r'] + + image_info['color_info']['mean_g'] + + image_info['color_info']['mean_b'] + ) / 3.0 + + if mean_intensity < 60: + lesion_assessment = 'high' + risk_score = 75 + elif mean_intensity < 100: + lesion_assessment = 'moderate' + risk_score = 50 + elif mean_intensity < 150: + lesion_assessment = 'low' + risk_score = 25 + else: + lesion_assessment = 'none' + risk_score = 10 + else: + lesion_assessment = 'none' + risk_score = 5 + + # Extract image dimensions for quality notes + img_width = image_info.get('width', 0) + img_height = image_info.get('height', 0) + resolution_category = image_info.get('quality_info', {}).get('resolution_category', 'unknown') + + # Assess image quality based on resolution + image_quality_sufficient = True + quality_note = "" + + if resolution_category in ['very_low', 'low']: + image_quality_sufficient = False + quality_note = f" Note: Image resolution ({img_width}x{img_height}) is below optimal for detailed analysis." + elif resolution_category == 'medium': + quality_note = f" Image resolution ({img_width}x{img_height}) is adequate for analysis." + else: + quality_note = f" Image resolution ({img_width}x{img_height}) is optimal for analysis." + + # Generate human-readable summary + summaries = { + 'none': f'{tz_type} transformation zone identified. No significant lesions detected. Routine screening recommended.{quality_note}', + 'low': f'{tz_type} transformation zone with minor acetowhite changes observed. Low-grade lesion suspected. Follow-up in 6 months recommended.{quality_note}', + 'moderate': f'{tz_type} transformation zone with acetowhite epithelium and irregular vascular patterns. Moderate-grade lesion suspected. Colposcopy and biopsy recommended.{quality_note}', + 'high': f'{tz_type} transformation zone with dense acetowhite areas and atypical vessels. High-grade lesion suspected. Immediate colposcopy and biopsy strongly recommended.{quality_note}' + } + + lesion_summary = summaries.get(lesion_assessment, 'Analysis inconclusive') + + # Return analysis results (width/height already in image_info, no need to duplicate) + return { + 'tz_type': tz_type, + 'lesion_assessment': lesion_assessment, + 'lesion_summary': lesion_summary, + 'risk_score': risk_score, + 'image_quality': resolution_category, + 'image_quality_sufficient': image_quality_sufficient + } + def _pre_process(self, inputs): """ Pre-process inputs: decode base64 images to numpy arrays. @@ -282,23 +387,20 @@ def _predict(self, inputs): # Extract image information image_info = self._extract_image_info(img_array) + # Generate cervical screening analysis + analysis = self._generate_cervical_analysis(img_array, image_info) + # Add processing metadata result = { 'index': idx, 'image_info': image_info, + 'analysis': analysis, 'processed_at': self.time(), 'processor_version': __VER__, 'model_name': 'cerviguard_image_analyzer', 'iteration': self._processed_count, } - # TODO: Future enhancement - call AI model for cervical cancer detection - # if self.has_ai_model(): - # ai_prediction = self.run_ai_model(img_array) - # result['ai_analysis'] = ai_prediction - # result['risk_level'] = ai_prediction['risk_level'] - # result['confidence'] = ai_prediction['confidence'] - results.append(result) return results diff --git a/ver.py b/ver.py index c60fb580..8de1b160 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.892' +__VER__ = '2.9.893' From aa2893615b87ff1e9c870e2a7c2839b4f6215638 Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+vitalii-t12@users.noreply.github.com> Date: Fri, 21 Nov 2025 14:03:39 +0200 Subject: [PATCH 04/11] Fix car restart (#311) * fix: container autostart policy * fix: shell injection vulnerability & remove deprecated flag * fix: add exponential backoff delay for tunnels * fix: autoupdate * feat: persist container stop state between plugin restarts * fix: paused log interval * fix: add numpy style docstrings * chore: inc version * fix: docstrings --- .../admin_container_app_runner.py | 48 +- .../container_apps/container_app_runner.py | 1593 +++++++++++++++-- .../container_apps/container_utils.py | 226 ++- .../container_apps/test_worker_app_runner.py | 34 + .../container_apps/worker_app_runner.py | 199 +- ver.py | 2 +- 6 files changed, 1865 insertions(+), 237 deletions(-) diff --git a/extensions/business/container_apps/admin_container_app_runner.py b/extensions/business/container_apps/admin_container_app_runner.py index d3939137..a5a04240 100644 --- a/extensions/business/container_apps/admin_container_app_runner.py +++ b/extensions/business/container_apps/admin_container_app_runner.py @@ -40,6 +40,15 @@ class AdminContainerAppRunnerPlugin( def on_init(self): + """ + Initialize the admin container app runner plugin. + + Calls parent initialization and ensures volumes dict is initialized. + + Returns + ------- + None + """ super(AdminContainerAppRunnerPlugin, self).on_init() if not self.volumes: self.volumes = {} @@ -49,7 +58,21 @@ def on_init(self): def _configure_volumes(self): """ - Processes the volumes specified in the configuration. + Process volume configuration for admin container. + + Configures volume mappings for the container, including optional edge node + data volume and user-specified volumes. Unlike the base class, this method + allows direct host path mapping without sandboxing. + + Returns + ------- + None + + Notes + ----- + - If MOUNT_EDGE_NODE_DATA_VOLUME is True, mounts /edge_node/_local_cache/_data + - All volumes are mounted with 'rw' (read-write) permissions + - Host paths are used directly without sanitization (admin-only feature) """ default_volume_rights = "rw" @@ -66,12 +89,29 @@ def _configure_volumes(self): def on_close(self): """ Lifecycle hook called when plugin is stopping. - Ensures container is shut down and logs are saved. - Ensures the log process is killed. - Stops tunnel if started. + + Ensures proper cleanup of container resources including: + - Container shutdown + - Log saving to disk + - Log process termination + - Tunnel termination if active + + Returns + ------- + None """ super(AdminContainerAppRunnerPlugin, self).on_close() def process(self): + """ + Main process loop for the admin container app runner. + + Delegates to parent class process method for container management, + health checks, and tunnel maintenance. + + Returns + ------- + None + """ super(AdminContainerAppRunnerPlugin, self).process() return \ No newline at end of file diff --git a/extensions/business/container_apps/container_app_runner.py b/extensions/business/container_apps/container_app_runner.py index d9fa8aab..ca9f1481 100644 --- a/extensions/business/container_apps/container_app_runner.py +++ b/extensions/business/container_apps/container_app_runner.py @@ -68,13 +68,75 @@ import time import socket import subprocess +from enum import Enum from naeural_core.business.base.web_app.base_tunnel_engine_plugin import BaseTunnelEnginePlugin as BasePlugin from extensions.business.mixins.chainstore_response_mixin import _ChainstoreResponseMixin from .container_utils import _ContainerUtilsMixin # provides container management support currently empty it is embedded in the plugin -__VER__ = "0.4.0" +__VER__ = "0.6.0" + +# Persistent state filename (general purpose) +_PERSISTENT_STATE_FILE = "container_persistent_state.pkl" + + +class ContainerState(Enum): + """Container lifecycle states for proper state machine management.""" + UNINITIALIZED = "uninitialized" # Container not yet created + STARTING = "starting" # Container is being launched + RUNNING = "running" # Container is running normally + STOPPING = "stopping" # Container is being stopped + STOPPED = "stopped" # Container stopped gracefully + FAILED = "failed" # Container crashed or exited with error + RESTARTING = "restarting" # Container is being restarted + PAUSED = "paused" # Manual pause requested + + +class StopReason(Enum): + """ + Reasons why a container stopped - used for restart policy decisions. + + Two categories of stop reasons: + 1. **Unplanned stops** (subject to RESTART_POLICY): + - CRASH, NORMAL_EXIT, HEALTH_CHECK_FAILED, UNKNOWN + - Policy determines if restart happens + + 2. **Planned restarts** (bypass RESTART_POLICY): + - IMAGE_UPDATE, CONFIG_UPDATE, EXTERNAL_UPDATE, MANUAL_STOP + - These trigger restarts via _perform_periodic_monitoring() + - Always executed regardless of policy (except MANUAL_STOP which pauses) + + Note: Subclasses can use EXTERNAL_UPDATE for domain-specific triggers + (e.g., Git updates, database migrations, file changes) + """ + # Unplanned stops + UNKNOWN = "unknown" + CRASH = "crash" # Container exited with non-zero code + NORMAL_EXIT = "normal_exit" # Container exited with code 0 + HEALTH_CHECK_FAILED = "health_check_failed" # Health check failures + + # Planned restarts + MANUAL_STOP = "manual_stop" # User requested stop via command + IMAGE_UPDATE = "image_update" # Restarting for image update + CONFIG_UPDATE = "config_update" # Restarting for config change + EXTERNAL_UPDATE = "external_update" # Generic external trigger (VCS, DB, file watch, etc.) + + +class RestartPolicy(Enum): + """ + Container restart policies (Docker-compatible). + + Policies: + NO: Never restart the container + ALWAYS: Always restart unless manually stopped + ON_FAILURE: Only restart on non-zero exit codes + UNLESS_STOPPED: Always restart unless explicitly stopped by user + """ + NO = "no" + ALWAYS = "always" + ON_FAILURE = "on-failure" + UNLESS_STOPPED = "unless-stopped" _CONFIG = { @@ -129,11 +191,25 @@ "ports": [] # dict of host_port: container_port mappings (e.g. {8080: 8081}) or list of container ports (e.g. [8080, 9000]) }, "USE_CUDA": False, # If True, will use nvidia runtime for GPU support - "RESTART_POLICY": "always", # "always" will restart the container if it stops + "RESTART_POLICY": "always", # "always", "on-failure", "unless-stopped", "no" "IMAGE_PULL_POLICY": "always", # "always" will always pull the image "AUTOUPDATE" : True, # If True, will check for image updates and pull them if available "AUTOUPDATE_INTERVAL": 100, + # Restart retry configuration (exponential backoff) + "RESTART_MAX_RETRIES": 5, # Max consecutive restart attempts before giving up (0 = unlimited) + "RESTART_BACKOFF_INITIAL": 2, # Initial backoff delay in seconds + "RESTART_BACKOFF_MAX": 300, # Maximum backoff delay in seconds (5 minutes) + "RESTART_BACKOFF_MULTIPLIER": 2, # Backoff multiplier for exponential backoff + "RESTART_RESET_INTERVAL": 300, # Reset retry count after this many seconds of successful run + + # Tunnel restart retry configuration (exponential backoff) + "TUNNEL_RESTART_MAX_RETRIES": 5, # Max consecutive tunnel restart attempts (0 = unlimited) + "TUNNEL_RESTART_BACKOFF_INITIAL": 2, # Initial tunnel backoff delay in seconds + "TUNNEL_RESTART_BACKOFF_MAX": 60, # Maximum tunnel backoff delay in seconds (1 minute) + "TUNNEL_RESTART_BACKOFF_MULTIPLIER": 2, # Tunnel backoff multiplier + "TUNNEL_RESTART_RESET_INTERVAL": 300, # Reset tunnel retry count after successful run + "VOLUMES": {}, # dict mapping host paths to container paths, e.g. {"/host/path": "/container/path"} "FILE_VOLUMES": {}, # dict mapping host paths to file configs: {"host_path": {"content": "...", "mounting_point": "..."}} @@ -145,6 +221,7 @@ "SHOW_LOG_EACH" : 60, # seconds to show logs "SHOW_LOG_LAST_LINES" : 5, # last lines to show "MAX_LOG_LINES" : 10_000, # max lines to keep in memory + "PAUSED_LOG_INTERVAL": 60, # seconds between paused state log messages # end of container-specific config options @@ -184,10 +261,25 @@ def port(self, value): def Pd(self, s, *args, score=-1, **kwargs): """ - Print a message to the console. + Print debug message if verbosity level allows. + + Parameters + ---------- + s : str + Message to print + score : int, optional + Verbosity threshold (default: -1). Message prints if cfg_car_verbose > score + *args + Additional positional arguments passed to P() + **kwargs + Additional keyword arguments passed to P() + + Returns + ------- + None """ if self.cfg_car_verbose > score: - s = "[DEPDBG] " + s + s = "[DEBUG] " + s self.P(s, *args, **kwargs) return @@ -196,7 +288,25 @@ def __reset_vars(self): self.container = None self.container_id = None self.container_name = self.cfg_instance_id + "_" + self.uuid(4) - self.docker_client = docker.from_env() + + # Initialize Docker client with proper error handling + try: + self.docker_client = docker.from_env() + # Verify Docker daemon is accessible by pinging it + self.docker_client.ping() + except docker.errors.DockerException as e: + raise RuntimeError( + f"Failed to connect to Docker daemon: {e}\n" + "Please ensure Docker is installed and running:\n" + " - Check: systemctl status docker (Linux) or Docker Desktop (Windows/Mac)\n" + " - Start: systemctl start docker (Linux) or start Docker Desktop\n" + " - Verify: docker ps" + ) from e + except Exception as e: + raise RuntimeError( + f"Unexpected error initializing Docker client: {e}\n" + "Please verify Docker installation and permissions." + ) from e self.container_logs = self.deque(maxlen=self.cfg_max_log_lines) @@ -208,7 +318,16 @@ def __reset_vars(self): self.env = {} self.dynamic_env = {} - self._is_manually_stopped = False # Flag to indicate if the container was manually stopped + # Container state machine + self.container_state = ContainerState.UNINITIALIZED + self.stop_reason = StopReason.UNKNOWN + + # Restart policy and retry logic + self._consecutive_failures = 0 + self._last_failure_time = 0 + self._next_restart_time = 0 + self._restart_backoff_seconds = 0 + self._last_successful_start = None # Initialize tunnel process self.tunnel_process = None @@ -220,6 +339,12 @@ def __reset_vars(self): self.extra_tunnel_configs = {} # Dict: {container_port: token} self.extra_tunnel_start_times = {} # Dict: {container_port: timestamp} + # Tunnel restart backoff tracking (per port) + self._tunnel_consecutive_failures = {} # Dict: {container_port: failure_count} + self._tunnel_last_failure_time = {} # Dict: {container_port: timestamp} + self._tunnel_next_restart_time = {} # Dict: {container_port: timestamp} + self._tunnel_last_successful_start = {} # Dict: {container_port: timestamp} + # Log streaming self.log_thread = None self.exec_threads = [] @@ -232,6 +357,7 @@ def __reset_vars(self): self._last_endpoint_check = 0 self._last_image_check = 0 self._last_extra_tunnels_ping = 0 + self._last_paused_log = 0 # Track when we last logged the paused message # Image update tracking self.current_image_hash = None @@ -244,11 +370,530 @@ def __reset_vars(self): return def _after_reset(self): - """Hook for subclasses to reset additional state.""" + """ + Hook for subclasses to reset additional state. + + Called after parent reset to allow subclasses to initialize + their own state variables. + + Returns + ------- + None + """ + return + + # ============================================================================ + # Persistent State Management (General Purpose) + # ============================================================================ + + def _load_persistent_state(self): + """ + Load persistent state from disk. + + Returns + ------- + dict + Persistent state dictionary (empty dict if no state exists) + """ + state = self.diskapi_load_pickle_from_data(_PERSISTENT_STATE_FILE) + return state if state is not None else {} + + def _save_persistent_state(self, **kwargs): + """ + Save or update persistent state fields. + + Parameters + ---------- + **kwargs + State fields to save/update (e.g., manually_stopped=True) + + Returns + ------- + None + + Examples + -------- + >>> self._save_persistent_state(manually_stopped=True, last_config_hash="abc123") + """ + # Load existing state + state = self._load_persistent_state() + # Update with new values + state.update(kwargs) + # Save back to disk + self.diskapi_save_pickle_to_data(state, _PERSISTENT_STATE_FILE) + return + + def _load_manual_stop_state(self): + """ + Load manual stop state from persistent storage. + + Returns + ------- + bool + True if container was manually stopped, False otherwise + """ + state = self._load_persistent_state() + return state.get("manually_stopped", False) + + def _clear_manual_stop_state(self): + """ + Clear manual stop state (called on RESTART command). + + Returns + ------- + None + """ + self._save_persistent_state(manually_stopped=False) + return + + # ============================================================================ + # End of Persistent State Management + # ============================================================================ + + # ============================================================================ + # Restart Policy and Retry Logic + # ============================================================================ + + def _normalize_restart_policy(self, policy): + """ + Normalize restart policy to RestartPolicy enum. + + Parameters + ---------- + policy : str, RestartPolicy, or None + Policy string, enum, or None + + Returns + ------- + RestartPolicy + Normalized RestartPolicy enum value + """ + if policy is None: + return RestartPolicy.NO + + # Already an enum + if isinstance(policy, RestartPolicy): + return policy + + # Convert string to enum (case-insensitive) + if isinstance(policy, str): + policy_str = policy.lower().strip() + try: + return RestartPolicy(policy_str) + except ValueError: + self.P(f"Unknown restart policy '{policy}', defaulting to 'no'", color='y') + return RestartPolicy.NO + + # Unknown type + self.P(f"Invalid restart policy type {type(policy)}, defaulting to 'no'", color='y') + return RestartPolicy.NO + + def _should_restart_container(self, stop_reason=None): + """ + Determine if container should be restarted based on RESTART_POLICY and stop reason. + + Implements Docker-style restart policies: + - NO: Never restart + - ALWAYS: Always restart (unless manually stopped) + - ON_FAILURE: Restart only on non-zero exit code + - UNLESS_STOPPED: Always restart unless explicitly stopped by user + + Parameters + ---------- + stop_reason : StopReason, optional + StopReason enum value indicating why container stopped + + Returns + ------- + bool + True if container should be restarted + """ + policy = self._normalize_restart_policy(self.cfg_restart_policy) + stop_reason = stop_reason or self.stop_reason + + # Never restart if manually stopped (user sent STOP command) + if stop_reason == StopReason.MANUAL_STOP: + self.Pd(f"Container manually stopped, restart policy '{policy.value}' will not trigger restart") + return False + + # Check if we're in PAUSED state + if self.container_state == ContainerState.PAUSED: + self.Pd("Container is paused, restart policy will not trigger restart") + return False + + # Policy: NO - never restart + if policy == RestartPolicy.NO: + return False + + # Policy: ALWAYS - restart unless manually stopped + if policy == RestartPolicy.ALWAYS: + return True + + # Policy: UNLESS_STOPPED - same as always in this implementation + if policy == RestartPolicy.UNLESS_STOPPED: + return True + + # Policy: ON_FAILURE - only restart on crashes + if policy == RestartPolicy.ON_FAILURE: + return stop_reason in [ + StopReason.CRASH, + StopReason.HEALTH_CHECK_FAILED, + StopReason.UNKNOWN, + ] + + # Fallback (should never reach here due to normalization) + self.P(f"Unhandled restart policy '{policy}', defaulting to no restart", color='y') + return False + + def _calculate_restart_backoff(self): + """ + Calculate exponential backoff delay for restart attempts. + + Returns + ------- + float + Seconds to wait before next restart attempt + """ + if self._consecutive_failures == 0: + return 0 + + # Exponential backoff: initial * (multiplier ^ (failures - 1)) + backoff = self.cfg_restart_backoff_initial * ( + self.cfg_restart_backoff_multiplier ** (self._consecutive_failures - 1) + ) + + # Cap at maximum backoff + backoff = min(backoff, self.cfg_restart_backoff_max) + + return backoff + + def _should_reset_retry_counter(self): + """ + Check if container has been running long enough to reset retry counter. + + Returns + ------- + bool + True if retry counter should be reset + """ + if not self._last_successful_start: + return False + + uptime = self.time() - self._last_successful_start + return uptime >= self.cfg_restart_reset_interval + + def _record_restart_failure(self): + """ + Record a restart failure and update backoff state. + + Returns + ------- + None + """ + self._consecutive_failures += 1 + self._last_failure_time = self.time() + self._restart_backoff_seconds = self._calculate_restart_backoff() + self._next_restart_time = self.time() + self._restart_backoff_seconds + + self.P( + f"Container restart failure #{self._consecutive_failures}. " + f"Next retry in {self._restart_backoff_seconds:.1f}s", + color='y' + ) + return + + def _record_restart_success(self): + """ + Record a successful restart and reset failure counters if appropriate. + + Returns + ------- + None + """ + self._last_successful_start = self.time() + + # Reset failure counter after first successful start + if self._consecutive_failures > 0: + self.P( + f"Container started successfully after {self._consecutive_failures} failure(s). " + f"Retry counter will reset after {self.cfg_restart_reset_interval}s of uptime.", + color='g' + ) + # Don't reset immediately - wait for reset interval + # self._consecutive_failures = 0 # This happens in _maybe_reset_retry_counter + # end if + return + + def _maybe_reset_retry_counter(self): + """ + Reset retry counter if container has been running successfully. + + Returns + ------- + None + """ + if self._consecutive_failures > 0 and self._should_reset_retry_counter(): + old_failures = self._consecutive_failures + self._consecutive_failures = 0 + self._restart_backoff_seconds = 0 + self.P( + f"Container running successfully for {self.cfg_restart_reset_interval}s. " + f"Reset failure counter (was {old_failures})", + color='g' + ) + # end if + return + + def _is_restart_backoff_active(self): + """ + Check if we're currently in backoff period. + + Returns + ------- + bool + True if we should wait before restarting + """ + if self._next_restart_time == 0: + return False + + current_time = self.time() + if current_time < self._next_restart_time: + remaining = self._next_restart_time - current_time + self.Pd(f"Restart backoff active: {remaining:.1f}s remaining") + return True + + return False + + def _has_exceeded_max_retries(self): + """ + Check if max retry attempts exceeded. + + Returns + ------- + bool + True if max retries exceeded (and max_retries > 0) + """ + if self.cfg_restart_max_retries <= 0: + return False # Unlimited retries + + return self._consecutive_failures >= self.cfg_restart_max_retries + + def _set_container_state(self, new_state, stop_reason=None): + """ + Update container state and optionally stop reason. + + Parameters + ---------- + new_state : ContainerState + ContainerState enum value + stop_reason : StopReason, optional + Optional StopReason enum value + + Returns + ------- + None + """ + old_state = self.container_state + self.container_state = new_state + + if stop_reason: + self.stop_reason = stop_reason + # end if + + self.Pd(f"Container state: {old_state.value} -> {new_state.value}", score=0) + return + + # ============================================================================ + # End of Restart Policy Logic + # ============================================================================ + + # ============================================================================ + # Tunnel Restart Backoff Logic + # ============================================================================ + + def _calculate_tunnel_backoff(self, container_port): + """ + Calculate exponential backoff delay for tunnel restart attempts. + + Parameters + ---------- + container_port : int + Container port for the tunnel + + Returns + ------- + float + Seconds to wait before next tunnel restart attempt + """ + failures = self._tunnel_consecutive_failures.get(container_port, 0) + if failures == 0: + return 0 + + # Exponential backoff: initial * (multiplier ^ (failures - 1)) + backoff = self.cfg_tunnel_restart_backoff_initial * ( + self.cfg_tunnel_restart_backoff_multiplier ** (failures - 1) + ) + + # Cap at maximum backoff + backoff = min(backoff, self.cfg_tunnel_restart_backoff_max) + + return backoff + + def _record_tunnel_restart_failure(self, container_port): + """ + Record a tunnel restart failure and update backoff state. + + Parameters + ---------- + container_port : int + Container port for the tunnel + + Returns + ------- + None + """ + self._tunnel_consecutive_failures[container_port] = \ + self._tunnel_consecutive_failures.get(container_port, 0) + 1 + self._tunnel_last_failure_time[container_port] = self.time() + + backoff = self._calculate_tunnel_backoff(container_port) + self._tunnel_next_restart_time[container_port] = self.time() + backoff + + failures = self._tunnel_consecutive_failures[container_port] + self.P( + f"Tunnel restart failure for port {container_port} (#{failures}). " + f"Next retry in {backoff:.1f}s", + color='y' + ) + return + + def _record_tunnel_restart_success(self, container_port): + """ + Record a successful tunnel restart. + + Parameters + ---------- + container_port : int + Container port for the tunnel + + Returns + ------- + None + """ + self._tunnel_last_successful_start[container_port] = self.time() + + # Note success if there were previous failures + failures = self._tunnel_consecutive_failures.get(container_port, 0) + if failures > 0: + self.P( + f"Tunnel for port {container_port} started successfully after {failures} failure(s).", + color='g' + ) + return + + def _is_tunnel_backoff_active(self, container_port): + """ + Check if tunnel is currently in backoff period. + + Parameters + ---------- + container_port : int + Container port for the tunnel + + Returns + ------- + bool + True if we should wait before restarting tunnel + """ + next_restart = self._tunnel_next_restart_time.get(container_port, 0) + if next_restart == 0: + return False + + current_time = self.time() + if current_time < next_restart: + remaining = next_restart - current_time + self.Pd(f"Tunnel {container_port} backoff active: {remaining:.1f}s remaining") + return True + + return False + + def _has_tunnel_exceeded_max_retries(self, container_port): + """ + Check if tunnel has exceeded max retry attempts. + + Parameters + ---------- + container_port : int + Container port for the tunnel + + Returns + ------- + bool + True if max retries exceeded (and max_retries > 0) + """ + if self.cfg_tunnel_restart_max_retries <= 0: + return False # Unlimited retries + + failures = self._tunnel_consecutive_failures.get(container_port, 0) + return failures >= self.cfg_tunnel_restart_max_retries + + def _maybe_reset_tunnel_retry_counter(self, container_port): + """ + Reset tunnel retry counter if it has been running successfully. + + Parameters + ---------- + container_port : int + Container port for the tunnel + + Returns + ------- + None + """ + failures = self._tunnel_consecutive_failures.get(container_port, 0) + if failures == 0: + return + + last_start = self._tunnel_last_successful_start.get(container_port, 0) + if not last_start: + return + + uptime = self.time() - last_start + if uptime >= self.cfg_tunnel_restart_reset_interval: + self.P( + f"Tunnel {container_port} running successfully for {self.cfg_tunnel_restart_reset_interval}s. " + f"Reset failure counter (was {failures})", + color='g' + ) + self._tunnel_consecutive_failures[container_port] = 0 + return + # ============================================================================ + # End of Tunnel Restart Backoff Logic + # ============================================================================ + def _normalize_container_command(self, value, *, field_name): - """Normalize a container command into a Docker-compatible representation.""" + """ + Normalize a container command into a Docker-compatible representation. + + Parameters + ---------- + value : str, list, tuple, or None + Command to normalize + field_name : str + Name of the configuration field (for error messages) + + Returns + ------- + str, list, or None + Normalized command ready for Docker + + Raises + ------ + ValueError + If command format is invalid + """ if value is None: return None @@ -265,7 +910,26 @@ def _normalize_container_command(self, value, *, field_name): raise ValueError(f"{field_name} must be None, a string, or a list/tuple of strings") def _normalize_command_sequence(self, value, *, field_name): - """Normalize build/run command sequences into a list of shell fragments.""" + """ + Normalize build/run command sequences into a list of shell fragments. + + Parameters + ---------- + value : str, list, tuple, or None + Command sequence to normalize + field_name : str + Name of the configuration field (for error messages) + + Returns + ------- + list of str + Normalized command list + + Raises + ------ + ValueError + If command sequence format is invalid + """ if value is None: return [] @@ -282,7 +946,18 @@ def _normalize_command_sequence(self, value, *, field_name): raise ValueError(f"{field_name} must be a string or an iterable of strings") def _validate_runner_config(self): - """Validate configuration and prepare normalized command data.""" + """ + Validate configuration and prepare normalized command data. + + Returns + ------- + None + + Raises + ------ + ValueError + If configuration is invalid + """ self._start_command = self._normalize_container_command( getattr(self, 'cfg_container_start_command', None), field_name='CONTAINER_START_COMMAND', @@ -297,15 +972,44 @@ def _validate_runner_config(self): return def _validate_subclass_config(self): - """Hook for subclasses to enforce additional validation.""" + """ + Hook for subclasses to enforce additional validation. + + Allows subclasses to add their own configuration validation + beyond the base container configuration checks. + + Returns + ------- + None + + Raises + ------ + ValueError + If subclass-specific validation fails + """ return def on_init(self): """ Lifecycle hook called once the plugin is initialized. - Authenticates with the container registry (if config is provided). - Determines whether Docker or Podman is available, sets up port (if needed), - and prepares for container run. + + Performs initial setup including: + - Container registry authentication + - Docker client initialization + - Dynamic environment variable configuration + - Resource limits and port allocation + - Volume and file volume configuration + - Extra tunnels validation + - Manual stop state checking + + Returns + ------- + None + + Raises + ------ + RuntimeError + If Docker daemon is not accessible or registry authentication fails """ self._reset_chainstore_response() self.__reset_vars() @@ -332,84 +1036,115 @@ def on_init(self): self._validate_runner_config() + # Check if container was manually stopped in a previous session + if self._load_manual_stop_state(): + self.P("Container was manually stopped in previous session. Keeping container paused.", color='y') + self._set_container_state(ContainerState.PAUSED, StopReason.MANUAL_STOP) + self._extra_on_init() self.P(f"{self.__class__.__name__} initialized (version {__VER__})", color='g') return def _extra_on_init(self): - """Hook for subclasses to perform additional initialization.""" + """ + Hook for subclasses to perform additional initialization. + + Called at the end of on_init() to allow subclasses to add + their own initialization logic. + + Returns + ------- + None + """ return def on_command(self, data, **kwargs): """ - Called when a INSTANCE_COMMAND is received by the plugin instance. - - The command is sent via `cmdapi_send_instance_command` from a commanding node (Deeploy plugin) - as in below simplified example: + Handle instance commands sent to the plugin. - ```python - pipeline = "some_app_pipeline" - signature = "CONTAINER_APP_RUNNER" - instance_id = "CONTAINER_APP_1e8dac" - node_address = "0xai_1asdfG11sammamssdjjaggxffaffaheASSsa" + Processes commands sent via cmdapi_send_instance_command from + commanding nodes. Supported commands: + - RESTART: Restart the container + - STOP: Stop container and enter paused state - instance_command = "RESTART" - - plugin.cmdapi_send_instance_command( - pipeline=pipeline, - signature=signature, - instance_id=instance_id, - instance_command=instance_command, - node_address=node_address, - ) - ``` - - while the `on_command` method should look like this: - - ```python - def on_command(self, data, **kwargs): - if data == "RESTART": - self.P("Restarting container...") - ... - elif data == "STOP": - self.P("Stopping container (restart policy still applies)...") - ... - else: - self.P(f"Unknown command: {data}") - return - ``` + Parameters + ---------- + data : str + Command string to execute + **kwargs + Additional command parameters + Returns + ------- + None + + Examples + -------- + Sending a command from another plugin: + >>> plugin.cmdapi_send_instance_command( + ... pipeline="app_pipeline", + ... signature="CONTAINER_APP_RUNNER", + ... instance_id="CONTAINER_APP_1e8dac", + ... instance_command="RESTART", + ... node_address="0xai_..." + ... ) """ self.P(f"Received a command: {data}") self.P(f"Command kwargs: {kwargs}") if data == "RESTART": self.P("Restarting container...") - self._is_manually_stopped = False + self._clear_manual_stop_state() # Clear persistent stop state + self._set_container_state(ContainerState.RESTARTING, StopReason.CONFIG_UPDATE) self._stop_container_and_save_logs_to_disk() - self._restart_container() + self._restart_container(StopReason.CONFIG_UPDATE) return elif data == "STOP": - self.P("Stopping container (restart policy still applies)...") + self.P("Stopping container (manual stop - restart policy will not trigger)...") + self._save_persistent_state(manually_stopped=True) # Save persistent stop state self._stop_container_and_save_logs_to_disk() - self._is_manually_stopped = True + self._set_container_state(ContainerState.PAUSED, StopReason.MANUAL_STOP) return else: self.P(f"Unknown plugin command: {data}") return def on_config(self, *args, **kwargs): - return self._handle_config_restart(self._restart_container) + """ + Lifecycle hook called when configuration changes. + + Stops current container and restarts with new configuration. + + Parameters + ---------- + *args + Positional arguments (unused) + **kwargs + Keyword arguments (unused) + + Returns + ------- + None + """ + return self._handle_config_restart(lambda: self._restart_container(StopReason.CONFIG_UPDATE)) def on_post_container_start(self): """ - Lifecycle hook called after the container is started. + Lifecycle hook called after container starts. + Runs commands in the container if specified in the config. + Called both after initial start and after restarts. + + Returns + ------- + None - - after the container first start - - after the container is restarted + Notes + ----- + This is a hook method that subclasses can override to add + custom post-start behavior. """ self.P("Container started, running post-start commands...") return @@ -419,7 +1154,19 @@ def on_post_container_start(self): def start_tunnel_engine(self): """ - Start the tunnel engine using the base tunnel engine functionality. + Start the main tunnel engine (Cloudflare or ngrok). + + Initiates tunnel process using base tunnel engine functionality + to expose container ports via public URL. + + Returns + ------- + None + + Notes + ----- + Only starts if TUNNEL_ENGINE_ENABLED is True. Tunnel type + is determined by use_cloudflare() method. """ if self.cfg_tunnel_engine_enabled: engine_name = "Cloudflare" if self.use_cloudflare() else "ngrok" @@ -429,11 +1176,19 @@ def start_tunnel_engine(self): self.P(f"{engine_name} tunnel started successfully", color='g') else: self.P(f"Failed to start {engine_name} tunnel", color='r') + # end if + # end if return def stop_tunnel_engine(self): """ - Stop the tunnel engine. + Stop the main tunnel engine. + + Terminates the running tunnel process and cleans up resources. + + Returns + ------- + None """ if self.tunnel_process: engine_name = "Cloudflare" if self.use_cloudflare() else "ngrok" @@ -441,14 +1196,20 @@ def stop_tunnel_engine(self): self.stop_tunnel_command(self.tunnel_process) self.tunnel_process = None self.P(f"{engine_name} tunnel stopped", color='g') + # end if return def get_tunnel_engine_ping_data(self): """ - Override to include extra tunnel URLs in payloads. + Get tunnel data including main app_url and extra tunnel URLs. - Returns: - dict: Tunnel data including main app_url and extra tunnel URLs + Returns + ------- + dict + Tunnel data including: + - app_url: Main tunnel URL (if available) + - extra_tunnel_urls: Dict mapping container ports to URLs + - extra_tunnel_status: Status of each extra tunnel """ result = {} @@ -478,7 +1239,13 @@ def get_tunnel_engine_ping_data(self): def maybe_extra_tunnels_ping(self): """ Emit periodic pings with extra tunnel URLs and status. - Similar to maybe_tunnel_engine_ping but for extra tunnels. + + Sends heartbeat payloads containing extra tunnel URLs and + their operational status at configured intervals. + + Returns + ------- + None """ if not self.extra_tunnel_urls: return @@ -519,11 +1286,15 @@ def _get_host_port_for_container_port(self, container_port): """ Get the host port mapped to a container port. - Args: - container_port: Container port (int) + Parameters + ---------- + container_port : int + Container port to look up - Returns: - int or None: Host port if found, None otherwise + Returns + ------- + int or None + Host port if found, None otherwise """ for host_port, c_port in self.extra_ports_mapping.items(): if c_port == container_port: @@ -534,19 +1305,34 @@ def _build_tunnel_command(self, container_port, token): """ Build Cloudflare tunnel command for a specific port. - Args: - container_port: Container port to tunnel - token: Cloudflare tunnel token + Parameters + ---------- + container_port : int + Container port to tunnel + token : str + Cloudflare tunnel token - Returns: - str or None: Command string to execute, or None if error + Returns + ------- + list or None + Command list to execute, or None if error """ host_port = self._get_host_port_for_container_port(container_port) if not host_port: self.P(f"No host port found for container port {container_port}", color='r') return None - return f"cloudflared tunnel --no-autoupdate run --token {token} --url http://127.0.0.1:{host_port}" + # Return list to avoid shell injection - use list-based subprocess + return [ + "cloudflared", + "tunnel", + "--no-autoupdate", + "run", + "--token", + str(token), + "--url", + f"http://127.0.0.1:{host_port}" + ] def _should_start_main_tunnel(self): """ @@ -557,8 +1343,10 @@ def _should_start_main_tunnel(self): 2. PORT is defined OR CLOUDFLARE_TOKEN is defined 3. Main PORT is not handled by EXTRA_TUNNELS - Returns: - bool: True if main tunnel should start + Returns + ------- + bool + True if main tunnel should start """ # Check if we have a token (backward compatibility) has_cloudflare_token = bool(getattr(self, 'cfg_cloudflare_token', None)) @@ -588,12 +1376,17 @@ def _start_extra_tunnel(self, container_port, token): """ Start a single extra tunnel for a specific container port. - Args: - container_port: Container port to expose - token: Cloudflare tunnel token + Parameters + ---------- + container_port : int + Container port to expose + token : str + Cloudflare tunnel token - Returns: - bool: True if tunnel started successfully, False otherwise + Returns + ------- + bool + True if tunnel started successfully, False otherwise """ if not token: self.P(f"No token provided for extra tunnel on port {container_port}", color='r') @@ -608,11 +1401,11 @@ def _start_extra_tunnel(self, container_port, token): try: host_port = self._get_host_port_for_container_port(container_port) self.P(f"Starting Cloudflare tunnel for container port {container_port} (host port {host_port})...", color='b') - self.Pd(f" Command: {command}") + self.Pd(f" Command: {' '.join(command)}") + # Use list-based subprocess to prevent shell injection process = subprocess.Popen( command, - shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0 @@ -630,15 +1423,34 @@ def _start_extra_tunnel(self, container_port, token): } self.extra_tunnel_start_times[container_port] = self.time() + # Record successful start for backoff tracking + self._record_tunnel_restart_success(container_port) + self.P(f"Extra tunnel for port {container_port} started (PID: {process.pid})", color='g') return True except Exception as e: self.P(f"Failed to start extra tunnel for port {container_port}: {e}", color='r') + # Record failure for backoff tracking + self._record_tunnel_restart_failure(container_port) return False def start_extra_tunnels(self): - """Start all configured extra tunnels.""" + """ + Start all configured extra Cloudflare tunnels. + + Iterates through extra_tunnel_configs and starts a tunnel + process for each configured port. + + Returns + ------- + None + + Notes + ----- + Logs the number of successfully started tunnels. + Failed tunnels are tracked for exponential backoff retry. + """ if not self.extra_tunnel_configs: self.Pd("No extra tunnels configured") return @@ -660,8 +1472,14 @@ def _stop_extra_tunnel(self, container_port): """ Stop a single extra tunnel. - Args: - container_port: Container port whose tunnel should be stopped + Parameters + ---------- + container_port : int + Container port whose tunnel should be stopped + + Returns + ------- + None """ process = self.extra_tunnel_processes.get(container_port) if not process: @@ -721,7 +1539,16 @@ def _stop_extra_tunnel(self, container_port): self.P(f"Error stopping extra tunnel for port {container_port}: {e}", color='r') def stop_extra_tunnels(self): - """Stop all extra tunnels.""" + """ + Stop all running extra tunnels. + + Iterates through all extra tunnel processes and stops each one, + reading remaining logs before termination. + + Returns + ------- + None + """ if not self.extra_tunnel_processes: return @@ -736,8 +1563,14 @@ def _read_extra_tunnel_logs(self, container_port): """ Read and process logs from an extra tunnel. - Args: - container_port: Container port whose tunnel logs to read + Parameters + ---------- + container_port : int + Container port whose tunnel logs to read + + Returns + ------- + None """ log_readers = self.extra_tunnel_log_readers.get(container_port, {}) @@ -765,12 +1598,21 @@ def _process_extra_tunnel_log(self, container_port, text, is_error=False): """ Process tunnel logs and extract URL. - For Cloudflare: Extract URL from pattern https://*.trycloudflare.com + For Cloudflare tunnels, extracts URL matching pattern + https://*.trycloudflare.com from the log output. - Args: - container_port: Container port - text: Log text - is_error: Whether this is error output + Parameters + ---------- + container_port : int + Container port for this tunnel + text : str + Log text to process + is_error : bool, optional + Whether this is error output (default: False) + + Returns + ------- + None """ log_prefix = f"[TUNNEL:{container_port}]" color = 'r' if is_error else 'd' @@ -791,7 +1633,16 @@ def _process_extra_tunnel_log(self, container_port, text, is_error=False): self.P(f"Extra tunnel URL for port {container_port}: {url}", color='g') def read_all_extra_tunnel_logs(self): - """Read logs from all extra tunnels.""" + """ + Read and process logs from all running extra tunnels. + + Iterates through all extra tunnel processes and reads their + stdout/stderr logs, extracting public URLs when found. + + Returns + ------- + None + """ for container_port in list(self.extra_tunnel_processes.keys()): try: self._read_extra_tunnel_logs(container_port) @@ -799,7 +1650,24 @@ def read_all_extra_tunnel_logs(self): self.Pd(f"Error reading logs for tunnel {container_port}: {e}") def start_container(self): - """Start the Docker container.""" + """ + Start the Docker container with configured settings. + + Creates and starts a Docker container with the configured image, + ports, volumes, environment variables, and resource limits. + + Returns + ------- + docker.models.containers.Container or None + Container object if started successfully, None otherwise + + Notes + ----- + Updates container state to RUNNING on success or FAILED on error. + Records restart success/failure for backoff tracking. + """ + self._set_container_state(ContainerState.STARTING) + log_str = f"Launching container with image '{self.cfg_image}'..." log_str += f"Container data:" @@ -840,20 +1708,41 @@ def start_container(self): self.cfg_image, **run_kwargs, ) + + self.container_id = self.container.short_id + self.P(f"Container started (ID: {self.container.short_id})", color='g') + + # Container started successfully + self._set_container_state(ContainerState.RUNNING) + self._record_restart_success() + + self._maybe_send_plugin_start_confirmation() + + return self.container + except Exception as e: self.P(f"Could not start container: {e}", color='r') self.container = None - return None + self._set_container_state(ContainerState.FAILED, StopReason.CRASH) + self._record_restart_failure() + return None - self.container_id = self.container.short_id - self.P(f"Container started (ID: {self.container.short_id})", color='g') + def stop_container(self): + """ + Stop and remove the Docker container. - self._maybe_send_plugin_start_confirmation() + Gracefully stops the container with a 5-second timeout, + then removes it from the Docker daemon. - return self.container + Returns + ------- + None - def stop_container(self): - """Stop and remove the Docker container if it is running.""" + Notes + ----- + If no container exists, logs a warning and returns. + Clears container and container_id attributes after removal. + """ if not self.container: self.P("No container to stop", color='y') return @@ -880,7 +1769,18 @@ def stop_container(self): return def _stream_logs(self, log_stream): - """Consume a log iterator from container logs and print its output.""" + """ + Consume a log iterator from container logs and print its output. + + Parameters + ---------- + log_stream : iterator + Log stream iterator from container.logs() + + Returns + ------- + None + """ if not log_stream: self.P("No log stream provided", color='y') return @@ -907,7 +1807,13 @@ def _stream_logs(self, log_stream): return def _start_container_log_stream(self): - """Start following container logs if not already streaming.""" + """ + Start following container logs if not already streaming. + + Returns + ------- + None + """ if not self.container: return @@ -927,17 +1833,47 @@ def _start_container_log_stream(self): return def _collect_exec_commands(self): - """Return the list of commands to execute inside the container.""" + """ + Return the list of commands to execute inside the container. + + Returns + ------- + list of str + Commands to execute, or empty list if none configured + """ return list(self._build_commands) if getattr(self, '_build_commands', None) else [] def _compose_exec_shell(self, commands): - """Compose a shell command string out of the command fragments.""" + """ + Compose a shell command string out of the command fragments. + + Parameters + ---------- + commands : list of str + Command fragments to chain together + + Returns + ------- + str or None + Shell command string with commands chained by &&, or None if empty + """ if not commands: return None return " && ".join(commands) def _run_container_exec(self, shell_cmd): - """Run a shell command inside the container and stream its output.""" + """ + Run a shell command inside the container and stream its output. + + Parameters + ---------- + shell_cmd : str + Shell command to execute inside container + + Returns + ------- + None + """ if not self.container or not shell_cmd: return @@ -961,7 +1897,13 @@ def _run_container_exec(self, shell_cmd): return def _maybe_execute_build_and_run(self): - """Execute configured build/run commands if necessary.""" + """ + Execute configured build/run commands if necessary. + + Returns + ------- + None + """ if self._commands_started: return @@ -981,6 +1923,18 @@ def _maybe_execute_build_and_run(self): return def _check_health_endpoint(self, current_time=None): + """ + Check health endpoint periodically if configured. + + Parameters + ---------- + current_time : float, optional + Current timestamp for interval checking + + Returns + ------- + None + """ if not self.container or not self.cfg_endpoint_url or self.cfg_endpoint_poll_interval <= 0: return @@ -991,7 +1945,13 @@ def _check_health_endpoint(self, current_time=None): return def _poll_endpoint(self): - """Poll the container's health endpoint and log the response.""" + """ + Poll the container's health endpoint and log the response. + + Returns + ------- + None + """ if not self.port: self.P("No port allocated, cannot poll endpoint", color='r') return @@ -1018,32 +1978,72 @@ def _poll_endpoint(self): return def _check_container_status(self): + """ + Check container status and update state machine. + + Returns + ------- + bool + True if container is running normally, False if stopped/failed + + Notes + ----- + Side effects: + - Updates container_state based on container status + - Sets stop_reason based on exit code + - Does NOT trigger restart - that's handled by process() + """ try: - if self.container: - # Refresh container status - self.container.reload() - if self.container.status != "running": - self.P( - f"Container stopped unexpectedly (exit code {self.container.attrs.get('State', {}).get('ExitCode')})", - color='r' - ) - self._commands_started = False - return False - # end if container not running - # end if self.container - return True + if not self.container: + return False + + # Refresh container status from Docker + # @see https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.Container.reload + self.container.reload() + + if self.container.status == "running": + # Container running normally + if self.container_state != ContainerState.RUNNING: + self._set_container_state(ContainerState.RUNNING) + return True + + # Container is not running - determine why + exit_code = self.container.attrs.get('State', {}).get('ExitCode', -1) + + # Determine stop reason based on exit code + if exit_code == 0: + stop_reason = StopReason.NORMAL_EXIT + else: + stop_reason = StopReason.CRASH + + # Update state + self._set_container_state(ContainerState.FAILED, stop_reason) + + self.P( + f"Container stopped (exit code: {exit_code}, reason: {stop_reason.value})", + color='y' if exit_code == 0 else 'r' + ) + + self._commands_started = False + return False + except Exception as e: self.P(f"Could not check container status: {e}", color='r') self.container = None self._commands_started = False - # end try - return False + self._set_container_state(ContainerState.FAILED, StopReason.UNKNOWN) + return False def _check_extra_tunnel_health(self): """ - Check health of extra tunnels and restart if needed. + Check health of extra tunnels and restart if needed with exponential backoff. + + Returns + ------- + None """ for container_port, process in list(self.extra_tunnel_processes.items()): + # Check if tunnel is still running if process.poll() is not None: # Process exited exit_code = process.returncode self.P(f"Extra tunnel for port {container_port} exited (code {exit_code})", color='r') @@ -1051,19 +2051,55 @@ def _check_extra_tunnel_health(self): # Clean up dead tunnel self._stop_extra_tunnel(container_port) - # Restart tunnel + # Record failure for backoff tracking + self._record_tunnel_restart_failure(container_port) + + # Check if we've exceeded max retries + if self._has_tunnel_exceeded_max_retries(container_port): + failures = self._tunnel_consecutive_failures.get(container_port, 0) + max_retries = self.cfg_tunnel_restart_max_retries + self.P( + f"Tunnel for port {container_port} restart abandoned after {failures} " + f"consecutive failures (max: {max_retries})", + color='r' + ) + continue + + # Check if we're in backoff period + if self._is_tunnel_backoff_active(container_port): + self.Pd(f"Tunnel {container_port} restart delayed due to active backoff period") + continue + + # All checks passed - attempt restart token = self.extra_tunnel_configs.get(container_port) if token: - self.P(f"Restarting extra tunnel for port {container_port}...", color='y') + failures = self._tunnel_consecutive_failures.get(container_port, 0) + self.P( + f"Restarting extra tunnel for port {container_port} (attempt {failures})...", + color='y' + ) self._start_extra_tunnel(container_port, token) + else: + # Tunnel is running - maybe reset retry counter + self._maybe_reset_tunnel_retry_counter(container_port) def _stop_container_and_save_logs_to_disk(self): """ - Stops the container and cloudflare tunnel. - Then logs are saved to disk. + Stop the container and all tunnels, then save logs to disk. + + Performs full shutdown sequence: + - Stops log streaming threads + - Stops main tunnel engine + - Stops all extra tunnels + - Stops and removes container + - Saves logs to disk + + Returns + ------- + None """ self.P(f"Stopping container app '{self.container_id}' ...") @@ -1105,9 +2141,16 @@ def _stop_container_and_save_logs_to_disk(self): def on_close(self): """ Lifecycle hook called when plugin is stopping. - Ensures container is shut down and logs are saved. - Ensures the log process is killed. - Stops tunnel if started. + + Performs cleanup including: + - Stopping container + - Stopping all tunnels (main and extra) + - Terminating log processes + - Saving container logs to disk + + Returns + ------- + None """ self._stop_container_and_save_logs_to_disk() @@ -1118,8 +2161,10 @@ def _get_local_image(self): """ Get the local Docker image if it exists. - Returns: - Image object or None if image doesn't exist locally + Returns + ------- + Image or None + Image object or None if image doesn't exist locally """ if not self.cfg_image: return None @@ -1134,11 +2179,15 @@ def _pull_image_from_registry(self): """ Pull image from registry (assumes authentication already done). - Returns: - Image object or None if pull failed + Returns + ------- + Image or None + Image object or None if pull failed - Raises: - RuntimeError: If authentication hasn't been performed + Raises + ------ + RuntimeError + If authentication hasn't been performed """ if not self.cfg_image: self.P("No Docker image configured", color='r') @@ -1169,11 +2218,15 @@ def _pull_image_with_fallback(self): 3. Falls back to local image if pull fails 4. Returns None only if both pull and local check fail - Returns: - Image object or None if no image is available + Returns + ------- + Image or None + Image object or None if no image is available - Raises: - RuntimeError: If authentication fails and no local image exists + Raises + ------ + RuntimeError + If authentication fails and no local image exists """ # Step 1: Authenticate with registry if not self._login_to_registry(): @@ -1205,11 +2258,15 @@ def _get_image_digest(self, img): """ Extract digest hash from image object. - Args: - img: Docker image object + Parameters + ---------- + img : Image + Docker image object - Returns: - str or None: Digest hash (sha256:...) or None + Returns + ------- + str or None + Digest hash (sha256:...) or None """ if not img: return None @@ -1254,11 +2311,15 @@ def _has_image_hash_changed(self, latest_hash): """ Check if image hash has changed from current version. - Args: - latest_hash: Latest image hash from registry + Parameters + ---------- + latest_hash : str + Latest image hash from registry - Returns: - bool: True if hash changed and update needed, False otherwise + Returns + ------- + bool + True if hash changed and update needed, False otherwise """ if not latest_hash: # Pull failed, can't determine if update needed @@ -1277,8 +2338,14 @@ def _handle_image_update(self, new_hash): """ Handle detected image update by updating hash and restarting container. - Args: - new_hash: New image hash detected + Parameters + ---------- + new_hash : str + New image hash detected + + Returns + ------- + None """ self.P(f"New image version detected ({new_hash} != {self.current_image_hash}). Restarting container...", color='y') @@ -1288,7 +2355,7 @@ def _handle_image_update(self, new_hash): self.current_image_hash = new_hash try: - self._restart_container() + self._restart_container(StopReason.IMAGE_UPDATE) except Exception as e: self.P(f"Container restart failed after image update: {e}", color='r') # Hash already updated, won't retry this version @@ -1304,8 +2371,14 @@ def _check_image_updates(self, current_time=None): 3. Compares with current hash 4. Triggers restart if changed - Args: - current_time: Current timestamp (for interval checking) + Parameters + ---------- + current_time : float, optional + Current timestamp (for interval checking) + + Returns + ------- + None """ if not self.cfg_autoupdate: return @@ -1330,11 +2403,39 @@ def _check_image_updates(self, current_time=None): return - def _restart_container(self): - """Restart the container from scratch.""" + def _restart_container(self, stop_reason=None): + """ + Restart the container from scratch. + + Parameters + ---------- + stop_reason : StopReason, optional + Optional StopReason enum indicating why restart was triggered + + Returns + ------- + None + """ self.P("Restarting container from scratch...", color='b') + + # Preserve state before reset (prevents redundant operations after restart) + preserved_failures = self._consecutive_failures + preserved_last_success = self._last_successful_start + preserved_last_image_check = self._last_image_check + preserved_current_hash = self.current_image_hash + self._stop_container_and_save_logs_to_disk() self.__reset_vars() + + # Restore preserved state (reset_vars clears it) + self._consecutive_failures = preserved_failures + self._last_successful_start = preserved_last_success + self._last_image_check = preserved_last_image_check + self.current_image_hash = preserved_current_hash + + # Set state after reset + self._set_container_state(ContainerState.RESTARTING, stop_reason or StopReason.UNKNOWN) + self._configure_dynamic_env() self._setup_resource_limits_and_ports() self._configure_volumes() @@ -1349,10 +2450,13 @@ def _restart_container(self): # Ensure image is available (respecting AUTOUPDATE and IMAGE_PULL_POLICY) if not self._ensure_image_available(): self.P("Failed to ensure image availability during restart, cannot start container", color='r') + self._set_container_state(ContainerState.FAILED, StopReason.CRASH) + self._record_restart_failure() return self.container = self.start_container() if not self.container: + # start_container already recorded the failure return self.container_start_time = self.time() @@ -1360,25 +2464,16 @@ def _restart_container(self): self._maybe_execute_build_and_run() return - def _ensure_image_with_autoupdate(self): - """ - Ensure image is available with autoupdate enabled. - Always pulls and tracks hash for version comparison. - - Returns: - bool: True if image available and hash tracked, False otherwise - """ - self.Pd("AUTOUPDATE enabled, pulling image and tracking hash") - self.current_image_hash = self._get_latest_image_hash() - return self.current_image_hash is not None - def _ensure_image_always_pull(self): """ Ensure image is available with 'always' pull policy. + Pulls image without tracking hash. - Returns: - bool: True if image pulled successfully, False otherwise + Returns + ------- + bool + True if image pulled successfully, False otherwise """ self.Pd("IMAGE_PULL_POLICY is 'always', pulling image") img = self._pull_image_with_fallback() @@ -1387,10 +2482,13 @@ def _ensure_image_always_pull(self): def _ensure_image_if_not_present(self): """ Ensure image is available with 'if-not-present' policy. + Only pulls if image doesn't exist locally. - Returns: - bool: True if image is available (locally or after pull), False otherwise + Returns + ------- + bool + True if image is available (locally or after pull), False otherwise """ # Check if image exists locally local_img = self._get_local_image() @@ -1408,16 +2506,20 @@ def _ensure_image_available(self): Ensure the container image is available before starting container. This method uses a strategy pattern based on configuration: - - AUTOUPDATE enabled: Always pull + track hash (update detection) + - AUTOUPDATE enabled: Ensure image exists locally (update detection handled separately) - IMAGE_PULL_POLICY='always': Always pull (no tracking) - IMAGE_PULL_POLICY='if-not-present' or default: Pull only if missing locally - Returns: - bool: True if image is available, False otherwise + Returns + ------- + bool + True if image is available, False otherwise """ # Strategy 1: AUTOUPDATE (takes precedence) + # When AUTOUPDATE is enabled, just ensure image exists locally + # Update checking and pulling happens in _check_image_updates() if self.cfg_autoupdate: - return self._ensure_image_with_autoupdate() + return self._ensure_image_if_not_present() # Strategy 2: Always pull policy if self.cfg_image_pull_policy == "always": @@ -1427,7 +2529,13 @@ def _ensure_image_available(self): return self._ensure_image_if_not_present() def _handle_initial_launch(self): - """Handle the initial container launch.""" + """ + Handle the initial container launch. + + Returns + ------- + None + """ try: self.P("Initial container launch...", color='b') @@ -1454,7 +2562,16 @@ def _handle_initial_launch(self): return def _perform_periodic_monitoring(self): - """Perform periodic monitoring tasks.""" + """ + Perform periodic monitoring tasks. + + Executes health checks, image update checks, tunnel health checks, + and any subclass-defined additional checks. + + Returns + ------- + None + """ current_time = self.time() self._check_health_endpoint(current_time) if self.cfg_autoupdate: @@ -1464,36 +2581,79 @@ def _perform_periodic_monitoring(self): if self.extra_tunnel_processes: self._check_extra_tunnel_health() - restart_required = self._perform_additional_checks(current_time) + restart_stop_reason = self._perform_additional_checks(current_time) - if restart_required: - self._restart_container() + if restart_stop_reason: + self._restart_container(restart_stop_reason) return def _perform_additional_checks(self, current_time): """ Hook for subclasses to implement additional monitoring checks. - + + This hook is called during periodic monitoring to check for conditions + that require container restart. Use StopReason.EXTERNAL_UPDATE for + domain-specific triggers (Git updates, database changes, file watches, etc.) + + Note: Restarts triggered here BYPASS restart policy - they always execute. + This is intentional for planned updates vs unplanned crashes. + Returns ------- - bool - True if container restart is required, False otherwise. + StopReason or None + StopReason if container restart is required, None otherwise. + + Examples + -------- + # Git-based updates (WorkerAppRunner) + def _perform_additional_checks(self, current_time): + if self._check_git_updates(): + return StopReason.EXTERNAL_UPDATE + return None + + # File watch updates + def _perform_additional_checks(self, current_time): + if self._config_file_changed(): + return StopReason.EXTERNAL_UPDATE + return None + + # Database schema updates + def _perform_additional_checks(self, current_time): + if self._schema_version_changed(): + return StopReason.EXTERNAL_UPDATE + return None """ - return False + return None def process(self): """ - This is the main process loop for the plugin that gets called each PROCESS_DELAY seconds and - it performs the following: + Main process loop for the plugin. - 1. Initialize and start tunnel engine if needed - 2. Check if container is running and restart if needed - 3. Perform periodic monitoring (health checks, etc.) - 4. Tunnel engine ping and maintenance + Called every PROCESS_DELAY seconds. Performs: + 1. Check for paused state (manual stop) + 2. Handle initial launch if container not started + 3. Initialize and start tunnel engine if needed + 4. Start main and extra tunnels + 5. Check container status and restart if needed + 6. Perform periodic monitoring (health checks, image updates) + 7. Send tunnel engine pings - """ - if self._is_manually_stopped: - self.Pd("Manually stopped app. Skipping launch...", color='y') + Returns + ------- + None + + Notes + ----- + The process loop implements a state machine for container lifecycle + management with automatic restart based on configured policies. + """ + # Use state machine instead of deprecated _is_manually_stopped flag + if self.container_state == ContainerState.PAUSED: + # Log paused message periodically instead of every process cycle + current_time = self.time() + if current_time - self._last_paused_log >= self.cfg_paused_log_interval: + self.P("Container is paused (manual stop). Send RESTART command to resume.", color='y') + self._last_paused_log = current_time return if not self.container: @@ -1516,9 +2676,50 @@ def process(self): if self.extra_tunnel_processes: self.read_all_extra_tunnel_logs() - if not self._check_container_status(): + # ============================================================================ + # Container Status Check and Restart Logic + # ============================================================================ + container_is_running = self._check_container_status() + + if not container_is_running: + # Container has stopped - decide if we should restart based on policy + policy = self._normalize_restart_policy(self.cfg_restart_policy) + + # Check if restart policy allows restart + if not self._should_restart_container(): + self.Pd(f"Container stopped. Restart policy '{policy.value}' does not allow restart.") + return + + # Check if we've exceeded max retry attempts + if self._has_exceeded_max_retries(): + self.P( + f"Container restart abandoned after {self._consecutive_failures} consecutive failures " + f"(max: {self.cfg_restart_max_retries})", + color='r' + ) + return + + # Check if we're in backoff period + if self._is_restart_backoff_active(): + self.Pd("Container restart delayed due to active backoff period") + return + + # All checks passed - attempt restart + self.P( + f"Container stopped. Restarting per policy '{policy.value}' " + f"(attempt {self._consecutive_failures + 1})", + color='y' + ) + self._restart_container(self.stop_reason) return + # Container is running normally - reset retry counter if appropriate + self._maybe_reset_retry_counter() + + # ============================================================================ + # End of Restart Logic + # ============================================================================ + self._start_container_log_stream() self._maybe_execute_build_and_run() diff --git a/extensions/business/container_apps/container_utils.py b/extensions/business/container_apps/container_utils.py index 38ca4b70..603cc5fc 100644 --- a/extensions/business/container_apps/container_utils.py +++ b/extensions/business/container_apps/container_utils.py @@ -16,7 +16,21 @@ class _ContainerUtilsMixin: ### START CONTAINER MIXIN METHODS ### def _handle_config_restart(self, restart_callable): - """Common handler to restart container instances when configuration changes.""" + """ + Handle container restart when configuration changes. + + Stops the current container and invokes the provided restart callable + to reinitialize with new configuration. + + Parameters + ---------- + restart_callable : callable + Function to call after stopping container to perform restart + + Returns + ------- + None + """ self.P(f"Received an updated config for {self.__class__.__name__}") self._stop_container_and_save_logs_to_disk() restart_callable() @@ -181,23 +195,58 @@ def _maybe_send_plugin_start_confirmation(self): def _setup_dynamic_env_var_host_ip(self): - """ Definition for `host_ip` dynamic env var type. """ + """ + Get host IP address for dynamic environment variable. + + Returns + ------- + str + The localhost IP address + """ return self.log.get_localhost_ip() def _setup_dynamic_env_var_some_other_calc_type(self): - """ Example definition for `some_other_calc_type` dynamic env var type. """ + """ + Example dynamic environment variable calculator. + + This is an example method showing how to implement custom dynamic + environment variable types. + + Returns + ------- + str + Example static value + """ return "some_other_value" def _configure_dynamic_env(self): """ - Set up dynamic environment variables based on the configuration. - - This method iterates over the `cfg_dynamic_env` dictionary, which contains - environment variable names as keys and a list of value parts as values. Each - value part specifies its type (e.g., "static" or "host_ip") and its value. - The method constructs the final value for each environment variable by - concatenating its parts. + Set up dynamic environment variables based on configuration. + + This method processes the cfg_dynamic_env dictionary, constructing + environment variable values by concatenating parts that can be either + static strings or dynamically computed values. + + Returns + ------- + None + + Notes + ----- + Dynamic parts are computed by calling methods named _setup_dynamic_env_var_{type}. + For example, a part with type "host_ip" calls _setup_dynamic_env_var_host_ip(). + + Examples + -------- + cfg_dynamic_env format: + { + "MY_VAR": [ + {"type": "static", "value": "prefix_"}, + {"type": "host_ip"}, + {"type": "static", "value": "_suffix"} + ] + } """ if len(self.cfg_dynamic_env): for variable_name, variable_value_list in self.cfg_dynamic_env.items(): @@ -439,7 +488,25 @@ def _setup_resource_limits_and_ports(self): return def _set_directory_permissions(self, path, mode=0o777): - """Ensure directory permissions allow non-root container access.""" + """ + Set directory permissions to allow non-root container access. + + Parameters + ---------- + path : str + Directory path to modify + mode : int, optional + Permission mode in octal notation (default: 0o777) + + Returns + ------- + None + + Notes + ----- + Failures are logged but do not raise exceptions. This is by design + to allow containers to attempt access even if permission changes fail. + """ try: os.chmod(path, mode) except PermissionError: @@ -649,7 +716,23 @@ def _setup_env_and_ports(self): return def _validate_container_config(self): - """Validate container configuration before starting.""" + """ + Validate container configuration before starting container. + + Checks that required configuration fields are present and properly + formatted, including IMAGE, CONTAINER_RESOURCES, and ENV. + + Returns + ------- + bool + Always returns True if validation passes + + Raises + ------ + ValueError + If IMAGE is missing, not a string, or if CONTAINER_RESOURCES + or ENV have invalid types + """ if not self.cfg_image: raise ValueError("IMAGE is required") @@ -669,7 +752,19 @@ def _validate_container_config(self): return True def _get_container_health_status(self, container=None): - """Get container health status using Docker client.""" + """ + Get container health status using Docker client. + + Parameters + ---------- + container : docker.models.containers.Container, optional + Container object to check. If None, uses self.container + + Returns + ------- + str + Container status: 'running', 'stopped', 'not_started', or 'error' + """ if container is None: container = getattr(self, 'container', None) @@ -685,7 +780,24 @@ def _get_container_health_status(self, container=None): def _validate_endpoint_config(self): - """Validate endpoint configuration for health checks.""" + """ + Validate endpoint configuration for health checks. + + Performs security and format validation on the configured + endpoint URL. + + Returns + ------- + bool + True if endpoint configuration is valid, False otherwise + + Notes + ----- + Validation checks include: + - URL is a string + - URL starts with '/' + - URL does not contain path traversal sequences (..) + """ if not hasattr(self, 'cfg_endpoint_url') or not self.cfg_endpoint_url: return False @@ -705,7 +817,19 @@ def _validate_endpoint_config(self): return True def _get_container_info(self): - """Get comprehensive container information.""" + """ + Get comprehensive container information. + + Collects container metadata including ID, status, ports, and volumes + into a single dictionary. + + Returns + ------- + dict + Container information with keys: container_id, container_name, + image, status, port, start_time, and optionally extra_ports + and volumes + """ container = getattr(self, 'container', None) info = { 'container_id': container.short_id if container else None, @@ -725,7 +849,16 @@ def _get_container_info(self): return info def _log_container_info(self): - """Log comprehensive container information.""" + """ + Log comprehensive container information to console. + + Formats and prints container metadata obtained from + _get_container_info() in a readable format. + + Returns + ------- + None + """ info = self._get_container_info() self.P("Container Information:", color='b') for key, value in info.items(): @@ -733,7 +866,19 @@ def _log_container_info(self): self.P(f" {key}: {value}", color='d') def _validate_port_allocation(self, port): - """Validate that a port is properly allocated.""" + """ + Validate that a port number is properly allocated. + + Parameters + ---------- + port : int + Port number to validate + + Returns + ------- + bool + True if port is valid (1-65535), False otherwise + """ if not port: return False @@ -746,7 +891,15 @@ def _validate_port_allocation(self, port): return True def _safe_get_container_stats(self): - """Safely get container statistics without raising exceptions.""" + """ + Safely get container statistics without raising exceptions. + + Returns + ------- + dict or None + Dictionary containing container stats (id, status, running, image, + created, ports) or None if container doesn't exist or error occurs + """ container = getattr(self, 'container', None) if not container: return None @@ -766,20 +919,39 @@ def _safe_get_container_stats(self): return None def _validate_docker_image_format(self, image_name): - """Validate Docker image name format.""" + """ + Validate Docker image name format. + + Parameters + ---------- + image_name : str + Docker image name to validate + + Returns + ------- + bool + True if image name format is valid, False otherwise + + Notes + ----- + Validation checks: + - Must be a string + - Must contain at least one ':' (tag) or '/' (repository) + - Must not contain whitespace characters + """ if not isinstance(image_name, str): return False - + # Basic validation - should contain at least one colon or slash if ':' not in image_name and '/' not in image_name: return False - + # Check for invalid characters invalid_chars = [' ', '\t', '\n', '\r'] for char in invalid_chars: if char in image_name: return False - + return True ### END COMMON CONTAINER UTILITY METHODS ### @@ -822,8 +994,14 @@ def _allocate_extra_tunnel_ports(self, container_ports): This handles the case where CONTAINER_RESOURCES["ports"] is empty but EXTRA_TUNNELS defines ports that need to be exposed. - Args: - container_ports: List of container ports to allocate + Parameters + ---------- + container_ports : list of int + List of container ports to allocate + + Returns + ------- + None """ self.P(f"Allocating host ports for {len(container_ports)} EXTRA_TUNNELS ports...", color='b') diff --git a/extensions/business/container_apps/test_worker_app_runner.py b/extensions/business/container_apps/test_worker_app_runner.py index ccfef000..9025faa5 100644 --- a/extensions/business/container_apps/test_worker_app_runner.py +++ b/extensions/business/container_apps/test_worker_app_runner.py @@ -86,6 +86,18 @@ def _install_dummy_base_plugin(): class WorkerAppRunnerConfigTests(unittest.TestCase): def _make_plugin(self): + """ + Create a mock WorkerAppRunnerPlugin instance for testing. + + Creates a plugin instance with mock attributes and methods + to facilitate unit testing without requiring actual Docker + infrastructure or network connections. + + Returns + ------- + WorkerAppRunnerPlugin + A mock plugin instance with test configuration + """ plugin = WorkerAppRunnerPlugin.__new__(WorkerAppRunnerPlugin) plugin.P = lambda *args, **kwargs: None from collections import deque @@ -115,6 +127,7 @@ def _make_plugin(self): return plugin def test_configure_repo_url_public(self): + """Test repository URL configuration for public repositories.""" plugin = self._make_plugin() plugin.cfg_vcs_data = { "REPO_OWNER": "ratio1", @@ -124,6 +137,7 @@ def test_configure_repo_url_public(self): self.assertEqual(plugin.repo_url, "https://github.com/ratio1/demo.git") def test_configure_repo_url_with_credentials(self): + """Test repository URL configuration with username and token credentials.""" plugin = self._make_plugin() plugin.cfg_vcs_data = { "REPO_OWNER": "ratio1", @@ -135,6 +149,7 @@ def test_configure_repo_url_with_credentials(self): self.assertEqual(plugin.repo_url, "https://user:token@github.com/ratio1/demo.git") def test_configure_repo_url_token_only(self): + """Test repository URL configuration with token-only authentication.""" plugin = self._make_plugin() plugin.cfg_vcs_data = { "REPO_OWNER": "ratio1", @@ -145,6 +160,7 @@ def test_configure_repo_url_token_only(self): self.assertEqual(plugin.repo_url, "https://token@github.com/ratio1/demo.git") def test_check_image_updates_respects_autoupdate_flag(self): + """Test that image update checks respect the AUTOUPDATE flag.""" plugin = self._make_plugin() plugin.cfg_autoupdate = False plugin._last_image_check = 0 @@ -156,6 +172,7 @@ def fail_pull(): plugin._check_image_updates(current_time=100) def test_check_image_updates_triggers_restart_on_new_digest(self): + """Test that new image digest triggers container restart.""" plugin = self._make_plugin() plugin.cfg_autoupdate = True plugin.cfg_autoupdate_interval = 10 @@ -171,6 +188,7 @@ def test_check_image_updates_triggers_restart_on_new_digest(self): self.assertEqual(restart_calls, ["called"]) def test_configure_volumes_primary_path(self): + """Test volume configuration creates directories with correct permissions.""" plugin = self._make_plugin() plugin.cfg_volumes = {"/data": "/app/data"} @@ -188,6 +206,7 @@ def test_configure_volumes_primary_path(self): self.assertEqual(stat.S_IMODE(os.stat(host_path).st_mode), 0o777) def test_on_config_triggers_restart(self): + """Test that configuration changes trigger container restart.""" plugin = self._make_plugin() with mock.patch.object(plugin, "_stop_container_and_save_logs_to_disk") as stop_mock, \ mock.patch.object(plugin, "_restart_from_scratch") as restart_mock: @@ -197,6 +216,7 @@ def test_on_config_triggers_restart(self): restart_mock.assert_called_once() def test__on_config_aliases_to_on_config(self): + """Test that _on_config is an alias for on_config method.""" plugin = self._make_plugin() with mock.patch.object(plugin, "_stop_container_and_save_logs_to_disk") as stop_mock, \ mock.patch.object(plugin, "_restart_from_scratch") as restart_mock: @@ -209,6 +229,18 @@ def test__on_config_aliases_to_on_config(self): class ContainerAppRunnerConfigTests(unittest.TestCase): def _make_plugin(self): + """ + Create a mock ContainerAppRunnerPlugin instance for testing. + + Creates a plugin instance with mock attributes and methods + to facilitate unit testing without requiring actual Docker + infrastructure or network connections. + + Returns + ------- + ContainerAppRunnerPlugin + A mock plugin instance with test configuration + """ plugin = ContainerAppRunnerPlugin.__new__(ContainerAppRunnerPlugin) plugin.P = lambda *args, **kwargs: None from collections import deque @@ -238,6 +270,7 @@ def _make_plugin(self): return plugin def test_on_config_triggers_restart(self): + """Test that configuration changes trigger container restart.""" plugin = self._make_plugin() with mock.patch.object(plugin, "_stop_container_and_save_logs_to_disk") as stop_mock, \ mock.patch.object(plugin, "_restart_container") as restart_mock: @@ -247,6 +280,7 @@ def test_on_config_triggers_restart(self): restart_mock.assert_called_once() def test__on_config_aliases_to_on_config(self): + """Test that _on_config is an alias for on_config method.""" plugin = self._make_plugin() with mock.patch.object(plugin, "_stop_container_and_save_logs_to_disk") as stop_mock, \ mock.patch.object(plugin, "_restart_container") as restart_mock: diff --git a/extensions/business/container_apps/worker_app_runner.py b/extensions/business/container_apps/worker_app_runner.py index d74d060c..3cfc6a99 100644 --- a/extensions/business/container_apps/worker_app_runner.py +++ b/extensions/business/container_apps/worker_app_runner.py @@ -6,16 +6,21 @@ - Runs build and run commands inside a container using ContainerAppRunner defaults - Clones a Git repository into the container before executing those commands - Monitors GitHub for new commits and restarts the container when changes land + - Uses StopReason.EXTERNAL_UPDATE for Git-triggered restarts (planned restarts) - Streams logs and manages tunnel lifecycle through the base runner """ import requests from urllib.parse import urlsplit -from extensions.business.container_apps.container_app_runner import ContainerAppRunnerPlugin +from extensions.business.container_apps.container_app_runner import ( + ContainerAppRunnerPlugin, + StopReason, + RestartPolicy, +) -__VER__ = "1.0.0" +__VER__ = "1.1.0" REPO_CLONE_PATH = "/app" @@ -57,12 +62,40 @@ class WorkerAppRunnerPlugin(ContainerAppRunnerPlugin): CONFIG = _CONFIG def Pd(self, s, *args, score=-1, **kwargs): + """ + Print debug message if verbosity level allows. + + Parameters + ---------- + s : str + Message to print + score : int, optional + Verbosity threshold (default: -1). Message prints if cfg_car_verbose > score + *args + Additional positional arguments passed to P() + **kwargs + Additional keyword arguments passed to P() + + Returns + ------- + None + """ if self.cfg_car_verbose > score: s = "[DEBUG] " + s self.P(s, *args, **kwargs) return def _after_reset(self): + """ + Reset worker-specific state variables. + + Called after parent reset to initialize Git-related state variables + for repository monitoring. + + Returns + ------- + None + """ super()._after_reset() self.current_commit = None self.branch = None @@ -74,6 +107,22 @@ def _after_reset(self): return def _validate_subclass_config(self): + """ + Validate WorkerAppRunner-specific configuration. + + Ensures BUILD_AND_RUN_COMMANDS and VCS_DATA are properly configured + for Git-based container deployment. + + Returns + ------- + None + + Raises + ------ + ValueError + If BUILD_AND_RUN_COMMANDS is empty, repository identification fails, + or POLL_INTERVAL is invalid + """ super()._validate_subclass_config() if not self._build_commands: @@ -99,6 +148,15 @@ def _validate_subclass_config(self): return def _extra_on_init(self): + """ + Perform worker-specific initialization. + + Ensures repository state is configured before container starts. + + Returns + ------- + None + """ super()._extra_on_init() self._ensure_repo_state(initial=True) return @@ -106,7 +164,22 @@ def _extra_on_init(self): # --- Command orchestration ------------------------------------------------- def _build_git_bootstrap_command(self): - """Return a shell snippet that installs git if it is missing in the container.""" + """ + Build shell command to install Git if missing in container. + + Creates a shell script that detects the container's package manager + and installs Git using the appropriate command. + + Returns + ------- + str + Shell command that checks for git and installs it if needed + + Notes + ----- + Supports package managers: apk, apt-get, apt, yum, dnf, microdnf, + pacman, and zypper. + """ installers = [ ("apk", "apk add --no-cache git openssh-client"), ("apt-get", "apt-get update && apt-get install -y git openssh-client"), @@ -130,6 +203,19 @@ def _build_git_bootstrap_command(self): return f"if ! command -v git >/dev/null 2>&1; then {inner_block} fi" def _collect_exec_commands(self): + """ + Collect commands to execute inside container. + + Builds command sequence that: + 1. Installs git if needed + 2. Clones repository + 3. Executes build/run commands + + Returns + ------- + list of str + Shell commands to execute, or empty list if repo not configured + """ base_commands = super()._collect_exec_commands() if not base_commands: return [] @@ -155,30 +241,43 @@ def _collect_exec_commands(self): # --- Monitoring ------------------------------------------------------------ def _perform_additional_checks(self, current_time): - """Check for git updates and return whether restart is required.""" + """ + Check for git updates and trigger restart if needed. + + Parameters + ---------- + current_time : float + Current timestamp for interval checking + + Returns + ------- + StopReason or None + StopReason.EXTERNAL_UPDATE if new commit detected, None otherwise + """ return self._check_git_updates(current_time) def _check_git_updates(self, current_time=None): """ Check for new commits in the repository. - + Returns ------- - bool - True if a new commit was detected and restart is required, False otherwise. + StopReason or None + StopReason.EXTERNAL_UPDATE if a new commit was detected and restart is required, + None otherwise. """ if not current_time: current_time = self.time() poll_interval = self._git_poll_interval if current_time - self._last_git_check < poll_interval: - return False + return None self._last_git_check = current_time latest_commit = self._get_latest_commit() if not latest_commit: - return False + return None if self.current_commit and latest_commit != self.current_commit: self.P( @@ -186,17 +285,25 @@ def _check_git_updates(self, current_time=None): color='y', ) self.current_commit = latest_commit - return True + return StopReason.EXTERNAL_UPDATE # Git update triggers external update restart else: if not self.current_commit: self.current_commit = latest_commit self.P(f"Commit check ({self.branch}): {latest_commit}", color='d') - return False + return None # --- Git helpers ----------------------------------------------------------- @property def _git_poll_interval(self): + """ + Get Git polling interval from configuration. + + Returns + ------- + int + Polling interval in seconds (minimum 15, default 60) + """ vcs_data = getattr(self, 'cfg_vcs_data', {}) or {} try: interval = int(vcs_data.get('POLL_INTERVAL', 60)) @@ -205,6 +312,20 @@ def _git_poll_interval(self): return max(interval, 15) def _ensure_repo_state(self, initial=False): + """ + Ensure repository state is configured. + + Configures branch, repository URL, and fetches latest commit. + + Parameters + ---------- + initial : bool, optional + If True, forces reconfiguration even if already configured (default: False) + + Returns + ------- + None + """ if self._repo_configured and not initial: return @@ -222,6 +343,22 @@ def _ensure_repo_state(self, initial=False): return def _configure_repo_url(self): + """ + Configure repository URL with authentication if provided. + + Builds GitHub repository URL with optional username/token credentials. + + Returns + ------- + None + + Notes + ----- + Sets self.repo_url with one of these formats: + - Public: https://github.com/owner/repo.git + - Token only: https://token@github.com/owner/repo.git + - User+token: https://user:token@github.com/owner/repo.git + """ vcs_data = getattr(self, 'cfg_vcs_data', {}) or {} username = vcs_data.get('USERNAME') token = vcs_data.get('TOKEN') @@ -246,6 +383,23 @@ def _configure_repo_url(self): return def _set_default_branch(self): + """ + Determine and set the default repository branch. + + Attempts to fetch the default branch from GitHub API if not + configured, falling back to 'main' if detection fails. + + Returns + ------- + None + + Notes + ----- + Branch selection priority: + 1. cfg_vcs_data['BRANCH'] if specified + 2. GitHub API default_branch if accessible + 3. 'main' as final fallback + """ vcs_data = getattr(self, 'cfg_vcs_data', {}) or {} repo_branch = vcs_data.get('BRANCH') repo_owner = self._repo_owner or vcs_data.get('REPO_OWNER') @@ -311,7 +465,28 @@ def _get_latest_commit(self, return_data=False): # --- Helpers --------------------------------------------------------------- def _extract_repo_identifier(self, vcs_data): - """Derive repository owner and name from VCS configuration.""" + """ + Extract repository owner and name from VCS configuration. + + Parses REPO_OWNER/REPO_NAME or extracts from REPO_URL. + + Parameters + ---------- + vcs_data : dict + VCS configuration dictionary + + Returns + ------- + tuple of (str, str) + Repository owner and name, or (None, None) if extraction fails + + Examples + -------- + >>> _extract_repo_identifier({'REPO_OWNER': 'user', 'REPO_NAME': 'repo'}) + ('user', 'repo') + >>> _extract_repo_identifier({'REPO_URL': 'https://github.com/user/repo.git'}) + ('user', 'repo') + """ repo_url = vcs_data.get('REPO_URL') owner = vcs_data.get('REPO_OWNER') name = vcs_data.get('REPO_NAME') diff --git a/ver.py b/ver.py index 8de1b160..59d37ff7 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.893' +__VER__ = '2.9.894' From 68aba2db6de32c834ca673b588f709ba3519833c Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+vitalii-t12@users.noreply.github.com> Date: Fri, 21 Nov 2025 18:48:40 +0200 Subject: [PATCH 05/11] CAR cpu/memory usage fixes (#312) * fix: move memory formatter functions to utils * fix: add memory and cpu limits for container app runner * fix: container start log * fix: rename config variable * fix: make distance between methods to 2 new lines * fix: add gpu support to CAR * chore: increment version * fix: container start log * fix: gpu logs cleanup * fix: docstring * fix: logs color cleanup * fix: scale mem reservation --- .../container_apps/container_app_runner.py | 292 +++++++++++------- .../container_apps/container_utils.py | 2 +- extensions/business/deeploy/deeploy_mixin.py | 78 +---- extensions/utils/memory_formatter.py | 80 +++++ ver.py | 2 +- 5 files changed, 273 insertions(+), 181 deletions(-) create mode 100644 extensions/utils/memory_formatter.py diff --git a/extensions/business/container_apps/container_app_runner.py b/extensions/business/container_apps/container_app_runner.py index ca9f1481..46772326 100644 --- a/extensions/business/container_apps/container_app_runner.py +++ b/extensions/business/container_apps/container_app_runner.py @@ -70,12 +70,16 @@ import subprocess from enum import Enum +from docker.types import DeviceRequest + from naeural_core.business.base.web_app.base_tunnel_engine_plugin import BaseTunnelEnginePlugin as BasePlugin from extensions.business.mixins.chainstore_response_mixin import _ChainstoreResponseMixin from .container_utils import _ContainerUtilsMixin # provides container management support currently empty it is embedded in the plugin -__VER__ = "0.6.0" +__VER__ = "0.6.1" + +from extensions.utils.memory_formatter import parse_memory_to_mb # Persistent state filename (general purpose) _PERSISTENT_STATE_FILE = "container_persistent_state.pkl" @@ -186,11 +190,10 @@ class RestartPolicy(Enum): "PORT": None, # internal container port if it's a web app (int) "CONTAINER_RESOURCES" : { "cpu": 1, # e.g. "0.5" for half a CPU, or "1.0" for one CPU core - "gpu": 0, + "gpu": 0, # 0 - no GPU, 1 - use GPU "memory": "512m", # e.g. "512m" for 512MB, "ports": [] # dict of host_port: container_port mappings (e.g. {8080: 8081}) or list of container ports (e.g. [8080, 9000]) }, - "USE_CUDA": False, # If True, will use nvidia runtime for GPU support "RESTART_POLICY": "always", # "always", "on-failure", "unless-stopped", "no" "IMAGE_PULL_POLICY": "always", # "always" will always pull the image "AUTOUPDATE" : True, # If True, will check for image updates and pull them if available @@ -221,7 +224,8 @@ class RestartPolicy(Enum): "SHOW_LOG_EACH" : 60, # seconds to show logs "SHOW_LOG_LAST_LINES" : 5, # last lines to show "MAX_LOG_LINES" : 10_000, # max lines to keep in memory - "PAUSED_LOG_INTERVAL": 60, # seconds between paused state log messages + # When container is STOPPED_MANUALLY (PAUSED state), this will define how often we log its existance + "PAUSED_STATE_LOG_INTERVAL": 60, # end of container-specific config options @@ -255,10 +259,12 @@ class ContainerAppRunnerPlugin( def port(self): return getattr(self, '_port', None) + @port.setter def port(self, value): self._port = value + def Pd(self, s, *args, score=-1, **kwargs): """ Print debug message if verbosity level allows. @@ -369,6 +375,7 @@ def __reset_vars(self): return + def _after_reset(self): """ Hook for subclasses to reset additional state. @@ -386,6 +393,7 @@ def _after_reset(self): # Persistent State Management (General Purpose) # ============================================================================ + def _load_persistent_state(self): """ Load persistent state from disk. @@ -398,6 +406,7 @@ def _load_persistent_state(self): state = self.diskapi_load_pickle_from_data(_PERSISTENT_STATE_FILE) return state if state is not None else {} + def _save_persistent_state(self, **kwargs): """ Save or update persistent state fields. @@ -423,6 +432,7 @@ def _save_persistent_state(self, **kwargs): self.diskapi_save_pickle_to_data(state, _PERSISTENT_STATE_FILE) return + def _load_manual_stop_state(self): """ Load manual stop state from persistent storage. @@ -435,6 +445,7 @@ def _load_manual_stop_state(self): state = self._load_persistent_state() return state.get("manually_stopped", False) + def _clear_manual_stop_state(self): """ Clear manual stop state (called on RESTART command). @@ -454,6 +465,7 @@ def _clear_manual_stop_state(self): # Restart Policy and Retry Logic # ============================================================================ + def _normalize_restart_policy(self, policy): """ Normalize restart policy to RestartPolicy enum. @@ -481,13 +493,14 @@ def _normalize_restart_policy(self, policy): try: return RestartPolicy(policy_str) except ValueError: - self.P(f"Unknown restart policy '{policy}', defaulting to 'no'", color='y') + self.P(f"Unknown restart policy '{policy}', defaulting to 'no'", color='r') return RestartPolicy.NO # Unknown type - self.P(f"Invalid restart policy type {type(policy)}, defaulting to 'no'", color='y') + self.P(f"Invalid restart policy type {type(policy)}, defaulting to 'no'", color='r') return RestartPolicy.NO + def _should_restart_container(self, stop_reason=None): """ Determine if container should be restarted based on RESTART_POLICY and stop reason. @@ -542,9 +555,10 @@ def _should_restart_container(self, stop_reason=None): ] # Fallback (should never reach here due to normalization) - self.P(f"Unhandled restart policy '{policy}', defaulting to no restart", color='y') + self.P(f"Unhandled restart policy '{policy}', defaulting to no restart", color='r') return False + def _calculate_restart_backoff(self): """ Calculate exponential backoff delay for restart attempts. @@ -567,6 +581,7 @@ def _calculate_restart_backoff(self): return backoff + def _should_reset_retry_counter(self): """ Check if container has been running long enough to reset retry counter. @@ -582,6 +597,7 @@ def _should_reset_retry_counter(self): uptime = self.time() - self._last_successful_start return uptime >= self.cfg_restart_reset_interval + def _record_restart_failure(self): """ Record a restart failure and update backoff state. @@ -598,10 +614,11 @@ def _record_restart_failure(self): self.P( f"Container restart failure #{self._consecutive_failures}. " f"Next retry in {self._restart_backoff_seconds:.1f}s", - color='y' + color='r' ) return + def _record_restart_success(self): """ Record a successful restart and reset failure counters if appropriate. @@ -617,13 +634,13 @@ def _record_restart_success(self): self.P( f"Container started successfully after {self._consecutive_failures} failure(s). " f"Retry counter will reset after {self.cfg_restart_reset_interval}s of uptime.", - color='g' ) # Don't reset immediately - wait for reset interval # self._consecutive_failures = 0 # This happens in _maybe_reset_retry_counter # end if return + def _maybe_reset_retry_counter(self): """ Reset retry counter if container has been running successfully. @@ -638,12 +655,12 @@ def _maybe_reset_retry_counter(self): self._restart_backoff_seconds = 0 self.P( f"Container running successfully for {self.cfg_restart_reset_interval}s. " - f"Reset failure counter (was {old_failures})", - color='g' + f"Reset failure counter (was {old_failures})" ) # end if return + def _is_restart_backoff_active(self): """ Check if we're currently in backoff period. @@ -664,6 +681,7 @@ def _is_restart_backoff_active(self): return False + def _has_exceeded_max_retries(self): """ Check if max retry attempts exceeded. @@ -678,6 +696,7 @@ def _has_exceeded_max_retries(self): return self._consecutive_failures >= self.cfg_restart_max_retries + def _set_container_state(self, new_state, stop_reason=None): """ Update container state and optionally stop reason. @@ -711,6 +730,7 @@ def _set_container_state(self, new_state, stop_reason=None): # Tunnel Restart Backoff Logic # ============================================================================ + def _calculate_tunnel_backoff(self, container_port): """ Calculate exponential backoff delay for tunnel restart attempts. @@ -739,6 +759,7 @@ def _calculate_tunnel_backoff(self, container_port): return backoff + def _record_tunnel_restart_failure(self, container_port): """ Record a tunnel restart failure and update backoff state. @@ -763,10 +784,11 @@ def _record_tunnel_restart_failure(self, container_port): self.P( f"Tunnel restart failure for port {container_port} (#{failures}). " f"Next retry in {backoff:.1f}s", - color='y' + color='r' ) return + def _record_tunnel_restart_success(self, container_port): """ Record a successful tunnel restart. @@ -786,11 +808,11 @@ def _record_tunnel_restart_success(self, container_port): failures = self._tunnel_consecutive_failures.get(container_port, 0) if failures > 0: self.P( - f"Tunnel for port {container_port} started successfully after {failures} failure(s).", - color='g' + f"Tunnel for port {container_port} started successfully after {failures} failure(s)." ) return + def _is_tunnel_backoff_active(self, container_port): """ Check if tunnel is currently in backoff period. @@ -817,6 +839,7 @@ def _is_tunnel_backoff_active(self, container_port): return False + def _has_tunnel_exceeded_max_retries(self, container_port): """ Check if tunnel has exceeded max retry attempts. @@ -837,6 +860,7 @@ def _has_tunnel_exceeded_max_retries(self, container_port): failures = self._tunnel_consecutive_failures.get(container_port, 0) return failures >= self.cfg_tunnel_restart_max_retries + def _maybe_reset_tunnel_retry_counter(self, container_port): """ Reset tunnel retry counter if it has been running successfully. @@ -863,7 +887,6 @@ def _maybe_reset_tunnel_retry_counter(self, container_port): self.P( f"Tunnel {container_port} running successfully for {self.cfg_tunnel_restart_reset_interval}s. " f"Reset failure counter (was {failures})", - color='g' ) self._tunnel_consecutive_failures[container_port] = 0 @@ -873,6 +896,7 @@ def _maybe_reset_tunnel_retry_counter(self, container_port): # End of Tunnel Restart Backoff Logic # ============================================================================ + def _normalize_container_command(self, value, *, field_name): """ Normalize a container command into a Docker-compatible representation. @@ -909,6 +933,7 @@ def _normalize_container_command(self, value, *, field_name): raise ValueError(f"{field_name} must be None, a string, or a list/tuple of strings") + def _normalize_command_sequence(self, value, *, field_name): """ Normalize build/run command sequences into a list of shell fragments. @@ -945,6 +970,7 @@ def _normalize_command_sequence(self, value, *, field_name): raise ValueError(f"{field_name} must be a string or an iterable of strings") + def _validate_runner_config(self): """ Validate configuration and prepare normalized command data. @@ -971,6 +997,7 @@ def _validate_runner_config(self): self._validate_subclass_config() return + def _validate_subclass_config(self): """ Hook for subclasses to enforce additional validation. @@ -989,6 +1016,7 @@ def _validate_subclass_config(self): """ return + def on_init(self): """ Lifecycle hook called once the plugin is initialized. @@ -1038,12 +1066,13 @@ def on_init(self): # Check if container was manually stopped in a previous session if self._load_manual_stop_state(): - self.P("Container was manually stopped in previous session. Keeping container paused.", color='y') + self.P("Container was manually stopped in previous session. Keeping container paused.") self._set_container_state(ContainerState.PAUSED, StopReason.MANUAL_STOP) self._extra_on_init() - self.P(f"{self.__class__.__name__} initialized (version {__VER__})", color='g') + self.P(f"{self.__class__.__name__} initialized (version {__VER__})") return + def _extra_on_init(self): """ @@ -1058,6 +1087,7 @@ def _extra_on_init(self): """ return + def on_command(self, data, **kwargs): """ Handle instance commands sent to the plugin. @@ -1110,6 +1140,7 @@ def on_command(self, data, **kwargs): self.P(f"Unknown plugin command: {data}") return + def on_config(self, *args, **kwargs): """ Lifecycle hook called when configuration changes. @@ -1150,8 +1181,6 @@ def on_post_container_start(self): return - - def start_tunnel_engine(self): """ Start the main tunnel engine (Cloudflare or ngrok). @@ -1170,16 +1199,17 @@ def start_tunnel_engine(self): """ if self.cfg_tunnel_engine_enabled: engine_name = "Cloudflare" if self.use_cloudflare() else "ngrok" - self.P(f"Starting {engine_name} tunnel...", color='b') + self.P(f"Starting {engine_name} tunnel...") self.tunnel_process = self.run_tunnel_engine() if self.tunnel_process: - self.P(f"{engine_name} tunnel started successfully", color='g') + self.P(f"{engine_name} tunnel started successfully") else: self.P(f"Failed to start {engine_name} tunnel", color='r') # end if # end if return + def stop_tunnel_engine(self): """ Stop the main tunnel engine. @@ -1192,13 +1222,14 @@ def stop_tunnel_engine(self): """ if self.tunnel_process: engine_name = "Cloudflare" if self.use_cloudflare() else "ngrok" - self.P(f"Stopping {engine_name} tunnel...", color='b') + self.P(f"Stopping {engine_name} tunnel...") self.stop_tunnel_command(self.tunnel_process) self.tunnel_process = None - self.P(f"{engine_name} tunnel stopped", color='g') + self.P(f"{engine_name} tunnel stopped") # end if return + def get_tunnel_engine_ping_data(self): """ Get tunnel data including main app_url and extra tunnel URLs. @@ -1236,6 +1267,7 @@ def get_tunnel_engine_ping_data(self): return result + def maybe_extra_tunnels_ping(self): """ Emit periodic pings with extra tunnel URLs and status. @@ -1282,6 +1314,7 @@ def maybe_extra_tunnels_ping(self): return + def _get_host_port_for_container_port(self, container_port): """ Get the host port mapped to a container port. @@ -1301,6 +1334,7 @@ def _get_host_port_for_container_port(self, container_port): return host_port return None + def _build_tunnel_command(self, container_port, token): """ Build Cloudflare tunnel command for a specific port. @@ -1334,6 +1368,7 @@ def _build_tunnel_command(self, container_port, token): f"http://127.0.0.1:{host_port}" ] + def _should_start_main_tunnel(self): """ Determine if the main tunnel should be started. @@ -1364,14 +1399,12 @@ def _should_start_main_tunnel(self): # If PORT is defined and in EXTRA_TUNNELS, skip main tunnel if self.cfg_port and self.cfg_port in self.extra_tunnel_configs: - self.P( - f"Main PORT {self.cfg_port} is defined in EXTRA_TUNNELS, using extra tunnel instead", - color='y' - ) + self.P(f"Main PORT {self.cfg_port} is defined in EXTRA_TUNNELS, using extra tunnel instead") return False return True + def _start_extra_tunnel(self, container_port, token): """ Start a single extra tunnel for a specific container port. @@ -1400,7 +1433,7 @@ def _start_extra_tunnel(self, container_port, token): # Start tunnel process try: host_port = self._get_host_port_for_container_port(container_port) - self.P(f"Starting Cloudflare tunnel for container port {container_port} (host port {host_port})...", color='b') + self.P(f"Starting Cloudflare tunnel for container port {container_port} (host port {host_port})...") self.Pd(f" Command: {' '.join(command)}") # Use list-based subprocess to prevent shell injection @@ -1426,7 +1459,7 @@ def _start_extra_tunnel(self, container_port, token): # Record successful start for backoff tracking self._record_tunnel_restart_success(container_port) - self.P(f"Extra tunnel for port {container_port} started (PID: {process.pid})", color='g') + self.P(f"Extra tunnel for port {container_port} started (PID: {process.pid})") return True except Exception as e: @@ -1435,6 +1468,7 @@ def _start_extra_tunnel(self, container_port, token): self._record_tunnel_restart_failure(container_port) return False + def start_extra_tunnels(self): """ Start all configured extra Cloudflare tunnels. @@ -1455,19 +1489,17 @@ def start_extra_tunnels(self): self.Pd("No extra tunnels configured") return - self.P(f"Starting {len(self.extra_tunnel_configs)} extra tunnel(s)...", color='b') + self.P(f"Starting {len(self.extra_tunnel_configs)} extra tunnel(s)...") started_count = 0 for container_port, token in self.extra_tunnel_configs.items(): if self._start_extra_tunnel(container_port, token): started_count += 1 - self.P( - f"Started {started_count}/{len(self.extra_tunnel_configs)} extra tunnels", - color='g' if started_count == len(self.extra_tunnel_configs) else 'y' - ) + self.P(f"Started {started_count}/{len(self.extra_tunnel_configs)} extra tunnels") return + def _stop_extra_tunnel(self, container_port): """ Stop a single extra tunnel. @@ -1486,7 +1518,7 @@ def _stop_extra_tunnel(self, container_port): return try: - self.P(f"Stopping extra tunnel for port {container_port}...", color='b') + self.P(f"Stopping extra tunnel for port {container_port}...") # Read remaining logs before stopping self._read_extra_tunnel_logs(container_port) @@ -1533,11 +1565,12 @@ def _stop_extra_tunnel(self, container_port): self.extra_tunnel_urls.pop(container_port, None) self.extra_tunnel_start_times.pop(container_port, None) - self.P(f"Extra tunnel for port {container_port} stopped", color='g') + self.P(f"Extra tunnel for port {container_port} stopped") except Exception as e: self.P(f"Error stopping extra tunnel for port {container_port}: {e}", color='r') + def stop_extra_tunnels(self): """ Stop all running extra tunnels. @@ -1552,12 +1585,13 @@ def stop_extra_tunnels(self): if not self.extra_tunnel_processes: return - self.P(f"Stopping {len(self.extra_tunnel_processes)} extra tunnel(s)...", color='b') + self.P(f"Stopping {len(self.extra_tunnel_processes)} extra tunnel(s)...") for container_port in list(self.extra_tunnel_processes.keys()): self._stop_extra_tunnel(container_port) - self.P("All extra tunnels stopped", color='g') + self.P("All extra tunnels stopped") + def _read_extra_tunnel_logs(self, container_port): """ @@ -1594,6 +1628,7 @@ def _read_extra_tunnel_logs(self, container_port): except Exception as e: self.Pd(f"Error reading stderr for tunnel {container_port}: {e}") + def _process_extra_tunnel_log(self, container_port, text, is_error=False): """ Process tunnel logs and extract URL. @@ -1615,12 +1650,13 @@ def _process_extra_tunnel_log(self, container_port, text, is_error=False): None """ log_prefix = f"[TUNNEL:{container_port}]" - color = 'r' if is_error else 'd' # Log the output for line in text.split('\n'): if line.strip(): self.Pd(f"{log_prefix} {line}", score=0) + # endif + # endfor line in text # Extract URL if not already found if container_port not in self.extra_tunnel_urls: @@ -1630,7 +1666,10 @@ def _process_extra_tunnel_log(self, container_port, text, is_error=False): if match: url = match.group(0) self.extra_tunnel_urls[container_port] = url - self.P(f"Extra tunnel URL for port {container_port}: {url}", color='g') + self.P(f"Extra tunnel URL for port {container_port}: {url}") + # endif + # endif container_port + return def read_all_extra_tunnel_logs(self): """ @@ -1649,6 +1688,7 @@ def read_all_extra_tunnel_logs(self): except Exception as e: self.Pd(f"Error reading logs for tunnel {container_port}: {e}") + def start_container(self): """ Start the Docker container with configured settings. @@ -1668,49 +1708,64 @@ def start_container(self): """ self._set_container_state(ContainerState.STARTING) - log_str = f"Launching container with image '{self.cfg_image}'..." - - log_str += f"Container data:" - log_str += f" Image: {self.cfg_image}" - log_str += f" Ports: {self.json_dumps(self.inverted_ports_mapping) if self.inverted_ports_mapping else 'None'}" - log_str += f" Env: {self.json_dumps(self.env) if self.env else 'None'}" - log_str += f" Volumes: {self.json_dumps(self.volumes) if self.volumes else 'None'}" - log_str += f" Resources: {self.json_dumps(self.cfg_container_resources) if self.cfg_container_resources else 'None'}" - log_str += f" Restart policy: {self.cfg_restart_policy}" - log_str += f" Pull policy: {self.cfg_image_pull_policy}" - log_str += f" Start command: {self._start_command if self._start_command else 'Image default'}" + log_str = f"Launching container with image '{self.cfg_image}'...\n" + + log_str += f"Container data:\n" + log_str += f" Image: {self.cfg_image}\n" + log_str += f" Ports: {self.json_dumps(self.inverted_ports_mapping) if self.inverted_ports_mapping else 'None'}\n" + log_str += f" Env: {self.json_dumps(self.env) if self.env else 'None'}\n" + log_str += f" Volumes: {self.json_dumps(self.volumes) if self.volumes else 'None'}\n" + log_str += f" Resources: {self.json_dumps(self.cfg_container_resources) if self.cfg_container_resources else 'None'}\n" + log_str += f" Restart policy: {self.cfg_restart_policy}\n" + log_str += f" Pull policy: {self.cfg_image_pull_policy}\n" + log_str += f" Start command: {self._start_command if self._start_command else 'Image default'}\n" + self.P(log_str) + nano_cpu_limit = self._cpu_limit * 1_000_000_000 + mem_reservation = f"{parse_memory_to_mb(self._mem_limit, 0.9)}m" + + run_kwargs = dict( + detach=True, + ports=self.inverted_ports_mapping, + environment=self.env, + volumes=self.volumes, + name=self.container_name, + nano_cpus=nano_cpu_limit, + mem_limit=self._mem_limit, + mem_reservation=mem_reservation, + # pids_limit= + ) + + if self._gpu_limit: + gpus_info = self.log.gpu_info() + if len(gpus_info) > 0: + self.P(f"GPU is requested and NVIDIA GPUs found, starting container with 1 GPU.") + run_kwargs["device_requests"] = [DeviceRequest( + count=1, # -1 = "all" devices + capabilities=[['gpu']], # what kind of device we want + # optionally: + # device_ids=['0', '1'], + # options={'compute': 'all'} + )] + else: + self.P("Warning! GPU is requested but no NVIDIA GPUs found, starting container without GPU") + # endif available GPUs + else: + self.P(f"Starting container without GPU") + # endif + try: - run_kwargs = dict( - detach=True, - ports=self.inverted_ports_mapping, - environment=self.env, - volumes=self.volumes, - name=self.container_name, - ) if self._start_command: run_kwargs['command'] = self._start_command - if self.cfg_use_cuda: - gpus_info = self.log.gpu_info() - if len(gpus_info) > 0: - run_kwargs['runtime'] = 'nvidia' - self.P(f"USE_CUDA is True and NVIDIA GPUs found, starting container with GPU support") - else: - self.P("Warning! USE_CUDA is True but no NVIDIA GPUs found, starting container without GPU support") - # endif available GPUs - else: - self.P(f"Starting container without GPU support") - # endif cfg_use_cuda - self.container = self.docker_client.containers.run( self.cfg_image, **run_kwargs, ) self.container_id = self.container.short_id - self.P(f"Container started (ID: {self.container.short_id})", color='g') + self.P(f"Container started (ID: {self.container.short_id})") # Container started successfully self._set_container_state(ContainerState.RUNNING) @@ -1727,6 +1782,7 @@ def start_container(self): self._record_restart_failure() return None + def stop_container(self): """ Stop and remove the Docker container. @@ -1744,22 +1800,22 @@ def stop_container(self): Clears container and container_id attributes after removal. """ if not self.container: - self.P("No container to stop", color='y') + self.P("No container to stop", color='r') return try: # Stop the container (gracefully) - self.P(f"Stopping container {self.container.short_id}...", color='b') + self.P(f"Stopping container {self.container.short_id}...") self.container.stop(timeout=5) - self.P(f"Container {self.container.short_id} stopped successfully", color='g') + self.P(f"Container {self.container.short_id} stopped successfully") except Exception as e: self.P(f"Error stopping container: {e}", color='r') # end try try: - self.P(f"Removing container {self.container.short_id}...", color='b') + self.P(f"Removing container {self.container.short_id}...") self.container.remove() - self.P(f"Container {self.container.short_id} removed successfully", color='g') + self.P(f"Container {self.container.short_id} removed successfully") except Exception as e: self.P(f"Error removing container: {e}", color='r') finally: @@ -1768,6 +1824,7 @@ def stop_container(self): # end try return + def _stream_logs(self, log_stream): """ Consume a log iterator from container logs and print its output. @@ -1782,7 +1839,7 @@ def _stream_logs(self, log_stream): None """ if not log_stream: - self.P("No log stream provided", color='y') + self.P("No log stream provided", color='r') return try: @@ -1792,20 +1849,21 @@ def _stream_logs(self, log_stream): try: log_str = log_bytes.decode("utf-8", errors="replace") except Exception as e: - self.P(f"Warning: Could not decode log bytes: {e}", color='y') + self.P(f"Warning: Could not decode log bytes: {e}", color='r') log_str = str(log_bytes) - self.P(f"[CONTAINER] {log_str}", color='d', end='') + self.P(f"[CONTAINER] {log_str}", end='') self.container_logs.append(log_str) if self._stop_event.is_set(): - self.P("Log streaming stopped by stop event", color='y') + self.P("Log streaming stopped by stop event") break except Exception as e: self.P(f"Exception while streaming logs: {e}", color='r') # end try return + def _start_container_log_stream(self): """ Start following container logs if not already streaming. @@ -1832,6 +1890,7 @@ def _start_container_log_stream(self): self.P(f"Could not start container log stream: {exc}", color='r') return + def _collect_exec_commands(self): """ Return the list of commands to execute inside the container. @@ -1843,6 +1902,7 @@ def _collect_exec_commands(self): """ return list(self._build_commands) if getattr(self, '_build_commands', None) else [] + def _compose_exec_shell(self, commands): """ Compose a shell command string out of the command fragments. @@ -1861,6 +1921,7 @@ def _compose_exec_shell(self, commands): return None return " && ".join(commands) + def _run_container_exec(self, shell_cmd): """ Run a shell command inside the container and stream its output. @@ -1878,7 +1939,7 @@ def _run_container_exec(self, shell_cmd): return try: - self.P(f"Running container exec command: {shell_cmd}", color='b') + self.P(f"Running container exec command: {shell_cmd}") exec_result = self.container.exec_run( ["sh", "-c", shell_cmd], stream=True, @@ -1896,6 +1957,7 @@ def _run_container_exec(self, shell_cmd): self._commands_started = False return + def _maybe_execute_build_and_run(self): """ Execute configured build/run commands if necessary. @@ -1922,6 +1984,7 @@ def _maybe_execute_build_and_run(self): self._run_container_exec(shell_cmd) return + def _check_health_endpoint(self, current_time=None): """ Check health endpoint periodically if configured. @@ -1944,6 +2007,7 @@ def _check_health_endpoint(self, current_time=None): # end if time elapsed return + def _poll_endpoint(self): """ Poll the container's health endpoint and log the response. @@ -1957,7 +2021,7 @@ def _poll_endpoint(self): return if not self.cfg_endpoint_url: - self.P("No endpoint URL configured, skipping health check", color='y') + self.P("No endpoint URL configured, skipping health check") return url = f"http://localhost:{self.port}{self.cfg_endpoint_url}" @@ -1967,7 +2031,7 @@ def _poll_endpoint(self): status = resp.status_code if status == 200: - self.P(f"Health check: {url} -> {status} OK", color='g') + self.P(f"Health check: {url} -> {status} OK") else: self.P(f"Health check: {url} -> {status} Error", color='r') except requests.RequestException as e: @@ -1977,6 +2041,7 @@ def _poll_endpoint(self): # end try return + def _check_container_status(self): """ Check container status and update state machine. @@ -2021,7 +2086,7 @@ def _check_container_status(self): self.P( f"Container stopped (exit code: {exit_code}, reason: {stop_reason.value})", - color='y' if exit_code == 0 else 'r' + color='r' if exit_code != 0 else 'b' ) self._commands_started = False @@ -2034,6 +2099,7 @@ def _check_container_status(self): self._set_container_state(ContainerState.FAILED, StopReason.UNKNOWN) return False + def _check_extra_tunnel_health(self): """ Check health of extra tunnels and restart if needed with exponential backoff. @@ -2076,7 +2142,7 @@ def _check_extra_tunnel_health(self): failures = self._tunnel_consecutive_failures.get(container_port, 0) self.P( f"Restarting extra tunnel for port {container_port} (attempt {failures})...", - color='y' + color='r' ) self._start_extra_tunnel(container_port, token) else: @@ -2084,8 +2150,6 @@ def _check_extra_tunnel_health(self): self._maybe_reset_tunnel_retry_counter(container_port) - - def _stop_container_and_save_logs_to_disk(self): """ Stop the container and all tunnels, then save logs to disk. @@ -2138,6 +2202,7 @@ def _stop_container_and_save_logs_to_disk(self): self.P(f"Failed to save logs: {exc}", color='r') return + def on_close(self): """ Lifecycle hook called when plugin is stopping. @@ -2175,6 +2240,7 @@ def _get_local_image(self): except Exception: return None + def _pull_image_from_registry(self): """ Pull image from registry (assumes authentication already done). @@ -2194,20 +2260,21 @@ def _pull_image_from_registry(self): return None try: - self.P(f"Pulling image '{self.cfg_image}'...", color='b') + self.P(f"Pulling image '{self.cfg_image}'...") img = self.docker_client.images.pull(self.cfg_image) # docker-py may return Image or list[Image] if isinstance(img, list) and img: img = img[-1] - self.P(f"Successfully pulled image '{self.cfg_image}'", color='g') + self.P(f"Successfully pulled image '{self.cfg_image}'") return img except Exception as e: self.P(f"Image pull failed: {e}", color='r') return None + def _pull_image_with_fallback(self): """ Pull image from registry with fallback to local image. @@ -2230,11 +2297,11 @@ def _pull_image_with_fallback(self): """ # Step 1: Authenticate with registry if not self._login_to_registry(): - self.P("Registry authentication failed", color='y') + self.P("Registry authentication failed", color='r') # Try to use local image if authentication fails local_img = self._get_local_image() if local_img: - self.P(f"Using local image (registry login failed): {self.cfg_image}", color='y') + self.P(f"Using local image (registry login failed): {self.cfg_image}", color='r') return local_img raise RuntimeError("Failed to authenticate with registry and no local image available.") @@ -2244,16 +2311,17 @@ def _pull_image_with_fallback(self): return img # Step 3: Fallback to local image - self.P(f"Pull failed, checking for local image: {self.cfg_image}", color='b') + self.P(f"Pull failed, checking for local image: {self.cfg_image}", color='r') local_img = self._get_local_image() if local_img: - self.P(f"Using local image as fallback: {self.cfg_image}", color='y') + self.P(f"Using local image as fallback: {self.cfg_image}", color='r') return local_img # Step 4: No image available self.P(f"No image available (pull failed and no local image): {self.cfg_image}", color='r') return None + def _get_image_digest(self, img): """ Extract digest hash from image object. @@ -2285,6 +2353,7 @@ def _get_image_digest(self, img): # Fallback to image id (sha256:...) return getattr(img, "id", None) + def _get_latest_image_hash(self): """ Get the latest identifier for the configured Docker image tag. @@ -2307,6 +2376,7 @@ def _get_latest_image_hash(self): img = self._pull_image_with_fallback() return self._get_image_digest(img) + def _has_image_hash_changed(self, latest_hash): """ Check if image hash has changed from current version. @@ -2327,13 +2397,14 @@ def _has_image_hash_changed(self, latest_hash): if not self.current_image_hash: # First time - establish baseline - self.P(f"Establishing baseline image hash: {latest_hash}", color='b') + self.P(f"Establishing baseline image hash: {latest_hash}") self.current_image_hash = latest_hash return False # Compare hashes return latest_hash != self.current_image_hash + def _handle_image_update(self, new_hash): """ Handle detected image update by updating hash and restarting container. @@ -2347,7 +2418,7 @@ def _handle_image_update(self, new_hash): ------- None """ - self.P(f"New image version detected ({new_hash} != {self.current_image_hash}). Restarting container...", color='y') + self.P(f"New image version detected ({new_hash} != {self.current_image_hash}). Restarting container...") # Update current_image_hash BEFORE restart # This prevents infinite retry loops if restart fails @@ -2359,7 +2430,8 @@ def _handle_image_update(self, new_hash): except Exception as e: self.P(f"Container restart failed after image update: {e}", color='r') # Hash already updated, won't retry this version - self.P(f"Image hash updated from {old_hash} to {new_hash}, but container restart failed", color='y') + self.P(f"Image hash updated from {old_hash} to {new_hash}, but container restart failed", color='r') + def _check_image_updates(self, current_time=None): """ @@ -2392,7 +2464,7 @@ def _check_image_updates(self, current_time=None): # Get latest image hash latest_hash = self._get_latest_image_hash() if not latest_hash: - self.P("Failed to check for image updates (pull failed). Container continues running.", color='y') + self.P("Failed to check for image updates (pull failed). Container continues running.") return # Check if update is needed @@ -2403,6 +2475,7 @@ def _check_image_updates(self, current_time=None): return + def _restart_container(self, stop_reason=None): """ Restart the container from scratch. @@ -2416,7 +2489,7 @@ def _restart_container(self, stop_reason=None): ------- None """ - self.P("Restarting container from scratch...", color='b') + self.P("Restarting container from scratch...") # Preserve state before reset (prevents redundant operations after restart) preserved_failures = self._consecutive_failures @@ -2464,6 +2537,7 @@ def _restart_container(self, stop_reason=None): self._maybe_execute_build_and_run() return + def _ensure_image_always_pull(self): """ Ensure image is available with 'always' pull policy. @@ -2479,6 +2553,7 @@ def _ensure_image_always_pull(self): img = self._pull_image_with_fallback() return img is not None + def _ensure_image_if_not_present(self): """ Ensure image is available with 'if-not-present' policy. @@ -2493,14 +2568,15 @@ def _ensure_image_if_not_present(self): # Check if image exists locally local_img = self._get_local_image() if local_img: - self.P(f"Image '{self.cfg_image}' found locally", color='g') + self.P(f"Image '{self.cfg_image}' found locally") return True # Image not found locally, pull it - self.P(f"Image not found locally, pulling '{self.cfg_image}'...", color='b') + self.P(f"Image not found locally, pulling '{self.cfg_image}'...") img = self._pull_image_with_fallback() return img is not None + def _ensure_image_available(self): """ Ensure the container image is available before starting container. @@ -2528,6 +2604,7 @@ def _ensure_image_available(self): # Strategy 3: If-not-present policy (default) return self._ensure_image_if_not_present() + def _handle_initial_launch(self): """ Handle the initial container launch. @@ -2537,7 +2614,7 @@ def _handle_initial_launch(self): None """ try: - self.P("Initial container launch...", color='b') + self.P("Initial container launch...") # Ensure image is available before starting container if not self._ensure_image_available(): @@ -2552,15 +2629,16 @@ def _handle_initial_launch(self): self._start_container_log_stream() self._maybe_execute_build_and_run() - self.P("Container launched successfully", color='g') + self.P("Container launched successfully") self.P(self.container) if self.current_image_hash: - self.P(f"Current image hash: {self.current_image_hash}", color='d') + self.P(f"Current image hash: {self.current_image_hash}") except Exception as e: self.P(f"Could not start container: {e}", color='r') # end try return + def _perform_periodic_monitoring(self): """ Perform periodic monitoring tasks. @@ -2587,6 +2665,7 @@ def _perform_periodic_monitoring(self): self._restart_container(restart_stop_reason) return + def _perform_additional_checks(self, current_time): """ Hook for subclasses to implement additional monitoring checks. @@ -2625,6 +2704,7 @@ def _perform_additional_checks(self, current_time): """ return None + def process(self): """ Main process loop for the plugin. @@ -2651,8 +2731,8 @@ def process(self): if self.container_state == ContainerState.PAUSED: # Log paused message periodically instead of every process cycle current_time = self.time() - if current_time - self._last_paused_log >= self.cfg_paused_log_interval: - self.P("Container is paused (manual stop). Send RESTART command to resume.", color='y') + if current_time - self._last_paused_log >= self.cfg_paused_state_log_interval: + self.P("Container is paused (manual stop). Send RESTART command to resume.") self._last_paused_log = current_time return @@ -2708,7 +2788,7 @@ def process(self): self.P( f"Container stopped. Restarting per policy '{policy.value}' " f"(attempt {self._consecutive_failures + 1})", - color='y' + color='r' ) self._restart_container(self.stop_reason) return diff --git a/extensions/business/container_apps/container_utils.py b/extensions/business/container_apps/container_utils.py index 603cc5fc..c1de0f79 100644 --- a/extensions/business/container_apps/container_utils.py +++ b/extensions/business/container_apps/container_utils.py @@ -363,7 +363,7 @@ def _setup_resource_limits_and_ports(self): container_resources = self.cfg_container_resources if isinstance(container_resources, dict) and len(container_resources) > 0: - self._cpu_limit = container_resources.get("cpu", DEFAULT_CPU_LIMIT) + self._cpu_limit = int(container_resources.get("cpu", DEFAULT_CPU_LIMIT)) self._gpu_limit = container_resources.get("gpu", DEFAULT_GPU_LIMIT) self._mem_limit = container_resources.get("memory", DEFAULT_MEM_LIMIT) diff --git a/extensions/business/deeploy/deeploy_mixin.py b/extensions/business/deeploy/deeploy_mixin.py index 650bec23..7906c363 100644 --- a/extensions/business/deeploy/deeploy_mixin.py +++ b/extensions/business/deeploy/deeploy_mixin.py @@ -7,6 +7,8 @@ DEEPLOY_RESOURCES, JOB_TYPE_RESOURCE_SPECS, WORKER_APP_RUNNER_SIGNATURE, JOB_APP_TYPES, JOB_APP_TYPES_ALL, \ CONTAINERIZED_APPS_SIGNATURES +from extensions.utils.memory_formatter import parse_memory_to_mb + DEEPLOY_DEBUG = True MESSAGE_PREFIX = "Please sign this message for Deeploy: " @@ -1022,76 +1024,6 @@ def deeploy_get_auth_result(self, inputs): } return result - # TODO: FIXME - def _format_memory_to_standard(self, memory_value): - """ - Convert memory value to standard format (string with unit). - Supports: "4096m", "4g", "4096", 4096 - - Args: - memory_value: Memory value as string or int - - Returns: - str: Standardized memory string (e.g., "4096m") - """ - if memory_value is None: - return None - - # If already a string with unit, return as-is - if isinstance(memory_value, str): - if memory_value.endswith(('m', 'M', 'g', 'G', 'k', 'K')): - return memory_value.lower() - # String number without unit - assume bytes, convert to MB - try: - bytes_value = int(memory_value) - return f"{bytes_value // (1024 * 1024)}m" - except ValueError: - return memory_value - - # If integer, assume bytes and convert to MB - if isinstance(memory_value, int): - return f"{memory_value // (1024 * 1024)}m" - - return str(memory_value) - - def _parse_memory_to_mb(self, memory_str): - """ - Parse memory string to megabytes. - - Args: - memory_str: Memory value like "4096m", "4g", "128m" - - Returns: - int: Memory in megabytes - """ - if memory_str is None: - return 0 - - memory_str = str(memory_str).lower().strip() - - # Extract number and unit - import re - match = re.match(r'^(\d+(?:\.\d+)?)\s*([kmg]?)$', memory_str) - if not match: - # Try to parse as plain number (assume MB) - try: - return int(float(memory_str)) - except ValueError: - return 0 - - value = float(match.group(1)) - unit = match.group(2) - - # Convert to MB - if unit == 'k': - return int(value / 1024) - elif unit == 'm' or unit == '': - return int(value) - elif unit == 'g': - return int(value * 1024) - - return 0 - def _aggregate_container_resources(self, inputs): """ Aggregate container resources across all CONTAINER_APP_RUNNER plugin instances. @@ -1136,7 +1068,7 @@ def _aggregate_container_resources(self, inputs): self.Pd(f" Container resources: cpu={cpu}, memory={memory}") total_cpu += cpu - memory_mb = self._parse_memory_to_mb(memory) + memory_mb = parse_memory_to_mb(memory) self.Pd(f" Parsed memory: {memory_mb}MB") total_memory_mb += memory_mb else: @@ -1314,10 +1246,10 @@ def deeploy_check_payment_and_job_owner(self, inputs, sender, is_create, debug=F expected_cpu_val = None requested_memory_mb = ( - None if requested_memory is None else self._parse_memory_to_mb(requested_memory) + None if requested_memory is None else parse_memory_to_mb(requested_memory) ) expected_memory_mb = ( - None if expected_memory is None else self._parse_memory_to_mb(expected_memory) + None if expected_memory is None else parse_memory_to_mb(expected_memory) ) self.Pd(f" Normalized: requested_cpu={requested_cpu_val}, expected_cpu={expected_cpu_val}") diff --git a/extensions/utils/memory_formatter.py b/extensions/utils/memory_formatter.py new file mode 100644 index 00000000..fae0c550 --- /dev/null +++ b/extensions/utils/memory_formatter.py @@ -0,0 +1,80 @@ +def format_memory_to_standard(self, memory_value): + """ + Convert memory value to standard format (string with unit). + Supports: "4096m", "4g", "4096", 4096 + + Args: + memory_value: Memory value as string or int + + Returns: + str: Standardized memory string (e.g., "4096m") + """ + if memory_value is None: + return None + + # If already a string with unit, return as-is + if isinstance(memory_value, str): + if memory_value.endswith(('m', 'M', 'g', 'G', 'k', 'K')): + return memory_value.lower() + # String number without unit - assume bytes, convert to MB + try: + bytes_value = int(memory_value) + return f"{bytes_value // (1024 * 1024)}m" + except ValueError: + return memory_value + + # If integer, assume bytes and convert to MB + if isinstance(memory_value, int): + return f"{memory_value // (1024 * 1024)}m" + + return str(memory_value) + + +def parse_memory_to_mb(memory_str, scaling_factor: float = 0.0): + """ + Parse memory string to megabytes. + + Args: + memory_str: Memory value like "4096m", "4g", "128m" + + Returns: + int: Memory in megabytes + """ + if memory_str is None: + return 0 + + memory_str = str(memory_str).lower().strip() + + # Extract number and unit + import re + match = re.match(r'^(\d+(?:\.\d+)?)\s*([bkmg]?)$', memory_str) + if not match: + # Try to parse as plain number (assume MB) + try: + return int(float(memory_str)) + except ValueError: + return 0 + + value = float(match.group(1)) + unit = match.group(2) + + if scaling_factor: + value = value * scaling_factor + + # Convert to MB + if unit == 'b': + return int(value / 1024 / 1000) + elif unit == 'k': + return int(value / 1024) + elif unit == 'm' or unit == '': + return int(value) + elif unit == 'g': + return int(value * 1024) + + return 0 + +if __name__=="__main__": + mem_samples = ["2000k", "512m", "1g"] + for mem_sample in mem_samples: + converted = parse_memory_to_mb(mem_sample, 0.9) + print(f"{mem_sample} -> {converted}") diff --git a/ver.py b/ver.py index 59d37ff7..89e7b140 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.894' +__VER__ = '2.9.895' From e7317cf816b29f9d49e232267978b7f1756d8a4e Mon Sep 17 00:00:00 2001 From: Alessandro <37877991+aledefra@users.noreply.github.com> Date: Thu, 27 Nov 2025 00:51:24 +0100 Subject: [PATCH 06/11] feat: new services job types (#314) --- extensions/business/deeploy/deeploy_const.py | 11 ++++------- ver.py | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/extensions/business/deeploy/deeploy_const.py b/extensions/business/deeploy/deeploy_const.py index c8c3184b..18318769 100644 --- a/extensions/business/deeploy/deeploy_const.py +++ b/extensions/business/deeploy/deeploy_const.py @@ -154,13 +154,6 @@ class DEFAULT_CONTAINER_RESOURCES: 7: {DEEPLOY_RESOURCES.CPU: 12, DEEPLOY_RESOURCES.MEMORY: '30g', DEEPLOY_RESOURCES.STORAGE: '120g'}, # high2 8: {DEEPLOY_RESOURCES.CPU: 16, DEEPLOY_RESOURCES.MEMORY: '62g', DEEPLOY_RESOURCES.STORAGE: '248g'}, # ultra1 9: {DEEPLOY_RESOURCES.CPU: 22, DEEPLOY_RESOURCES.MEMORY: '124g', DEEPLOY_RESOURCES.STORAGE: '496g'}, # ultra2 - # Services - 10: {DEEPLOY_RESOURCES.CPU: 1, DEEPLOY_RESOURCES.MEMORY: '2g', DEEPLOY_RESOURCES.STORAGE: '50g'}, # pgsql_low - 11: {DEEPLOY_RESOURCES.CPU: 2, DEEPLOY_RESOURCES.MEMORY: '4g', DEEPLOY_RESOURCES.STORAGE: '200g'}, # pgsql_med - 12: {DEEPLOY_RESOURCES.CPU: 1, DEEPLOY_RESOURCES.MEMORY: '2g', DEEPLOY_RESOURCES.STORAGE: '50g'}, # mysql_low - 13: {DEEPLOY_RESOURCES.CPU: 2, DEEPLOY_RESOURCES.MEMORY: '4g', DEEPLOY_RESOURCES.STORAGE: '200g'}, # mysql_med - 14: {DEEPLOY_RESOURCES.CPU: 1, DEEPLOY_RESOURCES.MEMORY: '2g', DEEPLOY_RESOURCES.STORAGE: '50g'}, # nosql_low - 15: {DEEPLOY_RESOURCES.CPU: 2, DEEPLOY_RESOURCES.MEMORY: '4g', DEEPLOY_RESOURCES.STORAGE: '200g'}, # nosql_med # Native Apps 16: {DEEPLOY_RESOURCES.CPU: 3, DEEPLOY_RESOURCES.MEMORY: '14g'}, # n_entry 17: {DEEPLOY_RESOURCES.CPU: 8, DEEPLOY_RESOURCES.MEMORY: '22g'}, # n_med1 @@ -188,6 +181,10 @@ class DEFAULT_CONTAINER_RESOURCES: 38: {DEEPLOY_RESOURCES.CPU: 16, DEEPLOY_RESOURCES.MEMORY: '62g', DEEPLOY_RESOURCES.STORAGE: '248g'}, # g_ultra + ultra1 39: {DEEPLOY_RESOURCES.CPU: 22, DEEPLOY_RESOURCES.MEMORY: '124g', DEEPLOY_RESOURCES.STORAGE: '496g'}, # g_ultra + ultra2 40: {DEEPLOY_RESOURCES.CPU: 22, DEEPLOY_RESOURCES.MEMORY: '124g'}, # g_ultra + n_ultra + # Services + 50: {DEEPLOY_RESOURCES.CPU: 1, DEEPLOY_RESOURCES.MEMORY: '2g', DEEPLOY_RESOURCES.STORAGE: '8g'}, # entry + 51: {DEEPLOY_RESOURCES.CPU: 2, DEEPLOY_RESOURCES.MEMORY: '4g', DEEPLOY_RESOURCES.STORAGE: '16g'}, # low1 + 52: {DEEPLOY_RESOURCES.CPU: 3, DEEPLOY_RESOURCES.MEMORY: '12g', DEEPLOY_RESOURCES.STORAGE: '48g'}, # high1 } class DEEPLOY_PLUGIN_DATA: diff --git a/ver.py b/ver.py index 89e7b140..51f0a27e 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.895' +__VER__ = '2.9.896' From 626525b6fa4779618922ff57b1697dd6d7927391 Mon Sep 17 00:00:00 2001 From: Andrei Ionut Damian Date: Thu, 27 Nov 2025 10:15:54 +0200 Subject: [PATCH 07/11] fix: (HOT) migrate CAR/WAR from "EE_" to "R1EN_" naming chore: prep for mainnet --- .../business/container_apps/container_utils.py | 13 +++++++++++++ ver.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/extensions/business/container_apps/container_utils.py b/extensions/business/container_apps/container_utils.py index c1de0f79..0eb5919d 100644 --- a/extensions/business/container_apps/container_utils.py +++ b/extensions/business/container_apps/container_utils.py @@ -93,14 +93,27 @@ def _get_default_env_vars(self): dct_env = { "CONTAINER_NAME": self.container_name, "EE_CONTAINER_NAME": self.container_name, + "R1EN_CONTAINER_NAME": self.container_name, "EE_HOST_IP": localhost_ip, + "R1EN_HOST_IP": localhost_ip, "EE_HOST_ID": self.ee_id, + "R1EN_HOST_ID": self.ee_id, "EE_HOST_ADDR": self.ee_addr, + "R1EN_HOST_ADDR": self.ee_addr, "EE_HOST_ETH_ADDR": self.bc.eth_address, + "R1EN_HOST_ETH_ADDR": self.bc.eth_address, "EE_EVM_NET": self.bc.get_evm_network(), + "R1EN_EVM_NET": self.bc.get_evm_network(), "EE_CHAINSTORE_API_URL": f"http://{localhost_ip}:31234", + "R1EN_CHAINSTORE_API_URL": f"http://{localhost_ip}:31234", "EE_R1FS_API_URL": f"http://{localhost_ip}:31235", + "R1EN_R1FS_API_URL": f"http://{localhost_ip}:31235", "EE_CHAINSTORE_PEERS": str_chainstore_peers, + "R1EN_CHAINSTORE_PEERS": str_chainstore_peers, + + # OBSERVATION: From now on only add new env vars with R1EN_ prefix + # to avoid missunderstandings with EE_ prefixed vars that + # are legacy from the Edge Node environment itself. } return dct_env diff --git a/ver.py b/ver.py index 51f0a27e..4e624df8 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.896' +__VER__ = '2.9.900' From 38b828270d881907b27ebac6f2cab517374f61c7 Mon Sep 17 00:00:00 2001 From: Cristi Bleotiu <164478159+cristibleotiu@users.noreply.github.com> Date: Thu, 27 Nov 2025 11:23:29 +0200 Subject: [PATCH 08/11] feat: support for sqlcoder and llm inference api (#313) * feat: support for sqlcoder and llm inference api * fix: removed date_string from prompt * fix: naming of endpoints * fix: ver change --- constants.py | 1 + .../inference_api/base_inference_api.py | 357 ++++++++++++++++++ extensions/business/jeeves/jeeves_api.py | 5 +- .../keysoft/keysoft_jeeves_constants.py | 17 +- extensions/business/mixins/nlp_agent_mixin.py | 25 +- extensions/business/nlp/vllm_agent.py | 62 ++- .../data/default/jeeves/jeeves_listener.py | 2 + extensions/serving/base/base_llm_serving.py | 16 +- .../default_inference/nlp/llama_sqlcoder.py | 68 ++++ .../nlp/llama_sqlcoder_small.py | 46 +++ .../serving/mixins_llm/llm_model_mixin.py | 22 +- .../serving/mixins_llm/llm_tokenizer_mixin.py | 29 +- extensions/serving/mixins_llm/llm_utils.py | 123 +++++- 13 files changed, 721 insertions(+), 52 deletions(-) create mode 100644 extensions/business/inference_api/base_inference_api.py create mode 100644 extensions/serving/default_inference/nlp/llama_sqlcoder.py create mode 100644 extensions/serving/default_inference/nlp/llama_sqlcoder_small.py diff --git a/constants.py b/constants.py index bae8e06e..ff85f5d8 100644 --- a/constants.py +++ b/constants.py @@ -160,6 +160,7 @@ class JeevesCt: JEEVES_API_SIGNATURES = [ "JEEVES_API", "KEYSOFT_JEEVES", + "BASE_INFERENCE_API", ] JEEVES_AGENT_SIGNATURES = [ diff --git a/extensions/business/inference_api/base_inference_api.py b/extensions/business/inference_api/base_inference_api.py new file mode 100644 index 00000000..3ad897f2 --- /dev/null +++ b/extensions/business/inference_api/base_inference_api.py @@ -0,0 +1,357 @@ +""" +LOCAL_SERVING_API Plugin + +This plugin creates a FastAPI server for both local-only access (localhost) and through tunneling +that works with a loopback data capture pipeline. +It can work with both async and sync requests. +In case of sync requests, they will be processed using PostponedRequest objects. +Otherwise, the request_id will be returned immediately, and the client can poll for results. + +Key Features: +- Loopback mode: Outputs return to DCT queue for processing +- Designed for LLM chat completions + +Available Endpoints: +- POST /create_chat_completion - Create chat completion (sync) +- POST /create_chat_completion_async - Create chat completion (async) +- GET /health - Health check +- GET /status_request - Check for current status of async request results + +Example pipeline configuration: +{ + "NAME": "local_inference_api", + "TYPE": "Loopback", + "PLUGINS": [ + { + "SIGNATURE": "BASE_INFERENCE_API", + "INSTANCES": [ + { + "INSTANCE_ID": "llm_interface", + "AI_ENGINE": "llama_cpp", + "STARTUP_AI_ENGINE_PARAMS": { + "HF_TOKEN": "", + "MODEL_FILENAME": "llama-3.2-1b-instruct-q4_k_m.gguf", + "MODEL_NAME": "hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF", + "SERVER_COLLECTOR_TIMEDELTA": 360000 + } + } + ] + } + ] +} +""" +from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin +from extensions.business.mixins.nlp_agent_mixin import _NlpAgentMixin, NLP_AGENT_MIXIN_CONFIG + + +__VER__ = '0.1.0' + +_CONFIG = { + **BasePlugin.CONFIG, + **NLP_AGENT_MIXIN_CONFIG, + + # MANDATORY SETTING IN ORDER TO RECEIVE REQUESTS + "ALLOW_EMPTY_INPUTS": True, # allow processing even when no input data is present + + # MANDATORY LOOPBACK SETTINGS + "IS_LOOPBACK_PLUGIN": True, + "TUNNEL_ENGINE_ENABLED": False, + "API_TITLE": "Local Inference API", + "API_SUMMARY": "FastAPI server for local-only inference.", + + "PROCESS_DELAY": 0, + "REQUEST_TIMEOUT": 600, # 10 minutes + "SAVE_PERIOD": 300, # 5 minutes + + "VALIDATION_RULES": { + **BasePlugin.CONFIG['VALIDATION_RULES'], + } +} + + +class BaseInferenceApiPlugin( + BasePlugin, + _NlpAgentMixin +): + CONFIG = _CONFIG + + def on_init(self): + super(BaseInferenceApiPlugin, self).on_init() + self._requests = {} + self._api_errors = {} + # This is different from self.last_error_time in BasePlugin + # self.last_error_time tracks unhandled errors that occur in the plugin loop + # This one tracks all errors that occur during API request handling + self.last_handled_error_time = None + self.last_persistence_save = 0 + self.load_persistence_data() + return + + """UTIL METHODS""" + if True: + def load_persistence_data(self): + cached_data = self.cacheapi_load_pickle() + if cached_data is not None: + # Useful only for debugging purposes + self._requests = cached_data.get('_requests', {}) + self._api_errors = cached_data.get('_api_errors', {}) + self.last_handled_error_time = cached_data.get('last_handled_error_time', None) + # endif cached_data is not None + return + + def maybe_save_persistence_data(self, force=False): + if force or (self.time() - self.last_persistence_save) > self.cfg_save_period: + data_to_save = { + '_requests': self._requests, + '_api_errors': self._api_errors, + 'last_handled_error_time': self.last_handled_error_time, + } + self.cacheapi_save_pickle(data_to_save) + self.last_persistence_save = self.time() + # endif needs saving + return + + def get_status(self): + last_error_time = self.last_handled_error_time + status = "ok" + if last_error_time is not None: + delta_seconds = (self.time() - last_error_time) + if delta_seconds < 300: + status = f"degraded (last error {int(delta_seconds)}s ago)" + # endif enough time has passed since last error + # endif last_error_time is not None + return status + + def solve_postponed_request(self, request_id: str): + if request_id in self._requests: + self.Pd(f"Checking status of request ID {request_id}...") + request_data = self._requests[request_id] + start_time = request_data.get("start_time", None) + timeout = request_data.get("timeout", self.cfg_request_timeout) + is_finished = request_data.get("finished", False) + if is_finished: + return request_data["result"] + elif start_time is not None and (self.time() - start_time) > timeout: + self.Pd(f"Request ID {request_id} has timed out after {timeout} seconds.") + error_response = f"Request ID {request_id} has timed out after {timeout} seconds." + request_data['result'] = { + "error": error_response, + "request_id": request_id, + } + request_data["finished"] = True + return request_data['result'] + # endif check finished or timeout + else: + self.Pd(f"Request ID {request_id} not found in requests.") + return { + "error": f"Request ID {request_id} not found." + } + # endif request exists + return self.create_postponed_request( + solver_method=self.solve_postponed_request, + method_kwargs={ + "request_id": request_id + } + ) + + def register_request( + self, + **kwargs + ): + request_id = self.uuid() + start_time = self.time() + request_data = { + **kwargs, + "request_id": request_id, + "start_time": start_time, + "finished": None, + "error": None, + } + self._requests[request_id] = request_data + return request_id, request_data + """END UTIL METHODS""" + + """GENERIC API ENDPOINTS""" + if True: + @BasePlugin.endpoint(method="GET") + def health(self): + return { + "status": self.get_status(), + "pipeline": self.get_stream_id(), + "plugin": self.get_signature(), + "instance_id": self.get_instance_id(), + "loopback_enabled": self.cfg_is_loopback_plugin, + "uptime": self.get_alive_time(), + "last_error_time": self.last_handled_error_time, + "total_errors": len(self._api_errors), + } + + @BasePlugin.endpoint(method="GET") + def check_request(self, request_id: str): + res = { + "error": f"Request ID {request_id} not found." + } + if request_id in self._requests: + request_data = self._requests[request_id] + is_finished = request_data.get("finished", False) + if is_finished: + res = request_data["result"] + else: + res = { + "status": "pending", + "request_id": request_id, + } + # endif request exists + return res + """END GENERIC API ENDPOINTS""" + + """CHAT COMPLETION SECTION""" + if True: + """VALIDATION SECTION""" + if True: + def check_messages(self, messages: list[dict]): + err_msg = None + if not isinstance(messages, list) or len(messages) == 0: + err_msg = "`messages` must be a non-empty list of message dicts." + if err_msg is None and not all(isinstance(m, dict) for m in messages): + err_msg = "Each message in `messages` must be a dict." + if err_msg is not None: + all_messages_valid = all( + isinstance(m, dict) and + 'role' in m and isinstance(m['role'], str) and + 'content' in m and isinstance(m['content'], str) + for m in messages + ) + if err_msg is None and not all_messages_valid: + err_msg = "Each message dict must contain 'role' (str) and 'content' (str) keys." + # endif err_msg is not None + return err_msg + + def check_chat_completion_params( + self, + messages: list[dict], + temperature: float = 0.7, + max_tokens: int = 512, + repeat_penalty: float = 1.0, + **kwargs + ): + err_msg = None + err_msg = self.check_messages(messages) + + return err_msg + """END VALIDATION SECTION""" + + def create_chat_completion_helper( + self, + messages: list[dict], + temperature: float = 0.7, + max_tokens: int = 512, + repeat_penalty: float = 1.0, + async_request=False, + **kwargs + ): + err_msg = self.check_chat_completion_params( + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + repeat_penalty=repeat_penalty, + **kwargs + ) + if err_msg is not None: + return { + "error": err_msg + } + # endif invalid params + request_id, request_data = self.register_request( + async_request=async_request, + **kwargs + ) + jeeves_content = { + 'REQUEST_ID': request_id, + **kwargs, + 'messages': messages, + 'temperature': temperature, + 'max_tokens': max_tokens, + 'repeat_penalty': repeat_penalty, + 'request_type': 'LLM', + } + self.Pd(f"Creating chat completion request {request_id} with data:\n{self.json_dumps(jeeves_content, indent=2)}") + self.add_payload_by_fields( + jeeves_content=jeeves_content, + signature=self.get_signature(), + ) + if async_request: + return { + "request_id": request_id, + "poll_url": f"/status_request?request_id={request_id}" + } + return self.solve_postponed_request(request_id=request_id) + + @BasePlugin.endpoint(method="POST") + def create_chat_completion( + self, + messages: list[dict], + temperature: float = 0.7, + max_tokens: int = 512, + repeat_penalty: float = 1.0, + **kwargs + ): + return self.create_chat_completion_helper( + async_request=False, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + repeat_penalty=repeat_penalty, + **kwargs + ) + + @BasePlugin.endpoint(method="POST") + def create_chat_completion_async( + self, + messages: list[dict], + temperature: float = 0.7, + max_tokens: int = 512, + repeat_penalty: float = 1.0, + **kwargs + ): + return self.create_chat_completion_helper( + async_request=True, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + repeat_penalty=repeat_penalty, + **kwargs + ) + """END CHAT COMPLETION SECTION""" + + def filter_valid_inference(self, inference): + is_valid = super(BaseInferenceApiPlugin, self).filter_valid_inference(inference=inference) + if is_valid: + request_id = inference.get('REQUEST_ID', None) + if request_id is None or request_id not in self._requests: + is_valid = False + # endif not is_valid + return is_valid + + def handle_single_inference(self, inference, model_name=None): + request_id = inference.get('REQUEST_ID', None) + self.Pd(f"Processing inference for request ID: {request_id}, model: {model_name}") + if request_id is None: + self.Pd("No REQUEST_ID found in inference; skipping.") + return + text_response = inference.get('text', None) + self._requests[request_id]['result'] = { + 'REQUEST_ID': request_id, + 'MODEL_NAME': model_name, + 'TEXT_RESPONSE': text_response, + } + self._requests[request_id]['finished'] = True + return + + def process(self): + self.maybe_save_persistence_data() + inferences = self.dataapi_struct_data_inferences() + self.handle_inferences(inferences=inferences) + return + + diff --git a/extensions/business/jeeves/jeeves_api.py b/extensions/business/jeeves/jeeves_api.py index bd27cb22..f050424c 100644 --- a/extensions/business/jeeves/jeeves_api.py +++ b/extensions/business/jeeves/jeeves_api.py @@ -476,7 +476,7 @@ def solve_postponed_request(self, request_id): If the request is not ready, it is further postponed. """ if request_id in self.__requests: - self.Pd(f"Checking request '{request_id}'...", color="yellow") + self.Pd(f"Checking request '{request_id}'...") request = self.__requests[request_id] start_time = request['start_time'] timeout = request['timeout'] @@ -492,6 +492,9 @@ def solve_postponed_request(self, request_id): # endif else: self.P(f"Request {request_id} not found in __requests.", color="red") + return { + 'error': f'Request ID {request_id} not found', + } # endif # Maybe handle case where request_id is not in __requests? return self.create_postponed_request( diff --git a/extensions/business/jeeves/partners/keysoft/keysoft_jeeves_constants.py b/extensions/business/jeeves/partners/keysoft/keysoft_jeeves_constants.py index 990a0981..6d8daa05 100644 --- a/extensions/business/jeeves/partners/keysoft/keysoft_jeeves_constants.py +++ b/extensions/business/jeeves/partners/keysoft/keysoft_jeeves_constants.py @@ -75,7 +75,7 @@ class KeysoftJeevesConstants: You respond with a complete SQL script exactly like the described pattern (comments + statements only). """ - SQL_INSTRUCTIONS_SIMPLE = """You are a SQL expert. + SQL_INSTRUCTIONS_SIMPLE_NO_EXAMPLE = """You are a SQL expert. ############################### # ABSOLUTE OUTPUT REQUIREMENTS ############################### @@ -102,8 +102,10 @@ class KeysoftJeevesConstants: 17. The following keywords are NOT allowed: ON REFERENCES 18. INSERT, UPDATE, ALTER, ADD, DELETE, SELECT, SET, or any DML statements are NOT allowed. -19. KEYWORDS MUST be separated from identifiers by AT LEAST one space. +19. KEYWORDS MUST be separated from identifiers by AT LEAST one space.""" + SQL_INSTRUCTIONS_SIMPLE = f""" +{SQL_INSTRUCTIONS_SIMPLE_NO_EXAMPLE} ############################### # VALIDATION EXAMPLE (ROLE DEMO) ############################### @@ -448,6 +450,17 @@ class KeysoftJeevesConstants: "temperature": 0.3, } }, + 'sql_simple_no_example': { + 'prompt': "file://_local_cache/sql_simple_instructions_no_example.txt", + 'prompt_default': SQL_INSTRUCTIONS_SIMPLE_NO_EXAMPLE, + 'additional_kwargs': { + # This may be re-enabled in the future. It was removed + # since now the generation ois deterministic + # 'valid_condition': "sql", + "process_method": "sql", + "temperature": 0.0, + } + }, 'sql_advanced': { 'prompt': SQL_INSTRUCTIONS_EXT, 'additional_kwargs': { diff --git a/extensions/business/mixins/nlp_agent_mixin.py b/extensions/business/mixins/nlp_agent_mixin.py index 5d63f10a..4c2af5f2 100644 --- a/extensions/business/mixins/nlp_agent_mixin.py +++ b/extensions/business/mixins/nlp_agent_mixin.py @@ -14,11 +14,14 @@ def Pd(self, msg, **kwargs): self.P(msg, **kwargs) return + def filter_valid_inference(self, inference): + return isinstance(inference, dict) and inference.get("IS_VALID", True) + def filter_valid_inferences(self, inferences, return_idxs=False): res = [] idxs = [] for idx, inf in enumerate(inferences): - if isinstance(inf, dict) and inf.get("IS_VALID", True): + if self.filter_valid_inference(inference=inf): res.append(inf) idxs.append(idx) # endfor inferences @@ -31,6 +34,17 @@ def inference_to_response(self, inference, model_name): 'TEXT_RESPONSE': inference.get('text'), } + def handle_single_inference(self, inference, model_name=None): + request_id = inference.get('REQUEST_ID', None) + self.Pd(f"Processing inference for request ID: {request_id}, model: {model_name}") + request_result = self.inference_to_response(inference, model_name) + current_payload_kwargs = { + 'result': request_result, + 'request_id': request_id, + } + self.add_payload_by_fields(**current_payload_kwargs) + return + def handle_inferences(self, inferences, data=None): if not isinstance(inferences, list): return @@ -49,14 +63,7 @@ def handle_inferences(self, inferences, data=None): # endif data is not None for inf in inferences: - request_id = inf.get('REQUEST_ID', None) - self.Pd(f"Processing inference for request ID: {request_id}, model: {model_name}") - request_result = self.inference_to_response(inf, model_name) - current_payload_kwargs = { - 'result': request_result, - 'request_id': request_id, - } - self.add_payload_by_fields(**current_payload_kwargs) + self.handle_single_inference(inference=inf, model_name=model_name) # endfor inferences return diff --git a/extensions/business/nlp/vllm_agent.py b/extensions/business/nlp/vllm_agent.py index e599cae7..77603d7a 100644 --- a/extensions/business/nlp/vllm_agent.py +++ b/extensions/business/nlp/vllm_agent.py @@ -28,6 +28,10 @@ "USE_GPU": None, + "GPU_MEMORY_UTILIZATION": 0.75, # 75% + "ALLOCATED_MEMORY": 12, # in GB + "CPU_CORES": 2, + "THREAD_MAX_WORKERS": 4, "DEFAULT_TEMPERATURE": 0.7, "DEFAULT_TOP_P": 0.9, @@ -53,6 +57,11 @@ class _ReqEntry: REQUESTS_MUTEX = "vllm_requests_mutex" DEFAULT_REQUEST_TIMEOUT = 60 # seconds +DEFAULT_GPU_MEMORY_UTILIZATION = 0.75 # 75% +GPU_MEMORY_UTILIZATION_MIN_VALUE = 0.6 # 60% +GPU_MEMORY_UTILIZATION_MAX_VALUE = 0.98 # 98% +DEFAULT_ALLOCATED_MEMORY = 12 # in GB +DEFAULT_NUM_CORES = 2 class VllmAgentPlugin(BasePlugin, _NlpAgentMixin): @@ -131,6 +140,32 @@ def get_hugging_face_api_token(self): env_token = self.os_environ.get("EE_HF_TOKEN", None) return configured_token or env_token or "" + def get_gpu_memory_utilization(self, show_logs: bool = False): + configured_utilization = self.cfg_gpu_memory_utilization + if isinstance(configured_utilization, (int, float)): + if GPU_MEMORY_UTILIZATION_MIN_VALUE <= configured_utilization <= GPU_MEMORY_UTILIZATION_MAX_VALUE: + return configured_utilization + else: + log_str = f"Invalid GPU_MEMORY_UTILIZATION value: {configured_utilization}. " + log_str += f"Must be between {GPU_MEMORY_UTILIZATION_MIN_VALUE} and {GPU_MEMORY_UTILIZATION_MAX_VALUE}." + if show_logs: + self.P(log_str) + # endif valid range + # endif valid type + return DEFAULT_GPU_MEMORY_UTILIZATION + + def get_allocated_memory(self): + configured_memory = self.cfg_allocated_memory + if isinstance(configured_memory, (int, float)) and configured_memory > 0: + return configured_memory + return DEFAULT_ALLOCATED_MEMORY # in GB + + def get_num_cpu_cores(self): + configured_cores = self.cfg_cpu_cores + if isinstance(configured_cores, int) and configured_cores > 0: + return configured_cores + return DEFAULT_NUM_CORES + """VLLM CONTAINER MANAGEMENT METHODS""" if True: def __get_all_used_ports(self): @@ -160,7 +195,8 @@ def get_start_command(self, port: int, model_name: str, use_gpu: bool): """ base_command = f"--host 0.0.0.0 --port {port} --model {model_name}" cpu_cmd_suffix = f"--dtype float16 --disable-frontend-multiprocessing --disable-async-output-proc" - gpu_cmd_suffix = f"--kv-cache-dtype fp8 --gpu-memory-utilization 0.75 --quantization bitsandbytes" + gpu_mem_util = self.get_gpu_memory_utilization(show_logs=False) + gpu_cmd_suffix = f"--kv-cache-dtype fp8 --gpu-memory-utilization {gpu_mem_util} --quantization bitsandbytes" cmd_suffix = gpu_cmd_suffix if use_gpu else cpu_cmd_suffix return f"{base_command} {cmd_suffix}" @@ -195,15 +231,15 @@ def compute_vllm_container_instance_config(self): res["PORT"] = self.container_port # TODO: review this + allocated_memory = self.get_allocated_memory() res["CONTAINER_RESOURCES"] = { - "cpu": 2, + "cpu": self.get_num_cpu_cores(), "gpu": 1 if use_gpu else 0, - "memory": "10g", + "memory": f"{allocated_memory}g", "ports": { str(self.container_port): str(self.container_port), } } - res["USE_CUDA"] = use_gpu res["IMAGE"] = "vllm/vllm-openai:latest" if use_gpu else "substratusai/vllm:main-cpu" res["ENV"] = { "HUGGING_FACE_HUB_TOKEN": self.get_hugging_face_api_token(), @@ -310,6 +346,13 @@ def maybe_start_vllm_container(self): self.persistence_save() return + def reset_container_state(self): + self.container_port = None + self.launched_container_config = None + self.launched_container_pipeline_name = None + self.persistence_save() + return + def maybe_clean_old_container_pipeline(self, pipeline_name: str = None): deletion_started = False pipeline_name = pipeline_name or self.pipeline_name_to_cleanup @@ -321,6 +364,7 @@ def maybe_clean_old_container_pipeline(self, pipeline_name: str = None): current_pipeline_names = [p["NAME"] for p in current_node_pipeline] if pipeline_name not in current_pipeline_names: self.P(f"vLLM container pipeline: {pipeline_name} not found among current pipelines, assuming already deleted.") + self.reset_container_state() return deletion_started self.P(f"Stopping vLLM container pipeline: {pipeline_name}") self.cmdapi_stop_pipeline( @@ -332,10 +376,7 @@ def maybe_clean_old_container_pipeline(self, pipeline_name: str = None): extracted_pipeline_name = (self.launched_container_config or {}).get("NAME", None) launched_pipeline_name = launched_pipeline_name or extracted_pipeline_name if pipeline_name == launched_pipeline_name: - self.launched_container_config = None - self.launched_container_pipeline_name = None - self.container_port = None - self.persistence_save() + self.reset_container_state() # endif launched container deletion_started = True return deletion_started @@ -587,10 +628,7 @@ def on_close(self): node_address=None, name=self.launched_container_pipeline_name ) - self.launched_container_pipeline_name = None - self.launched_container_config = None - self.container_port = None - self.persistence_save() + self.reset_container_state() # endif launched container pipeline super(VllmAgentPlugin, self).on_close() return diff --git a/extensions/data/default/jeeves/jeeves_listener.py b/extensions/data/default/jeeves/jeeves_listener.py index 1de74d0f..b523769e 100644 --- a/extensions/data/default/jeeves/jeeves_listener.py +++ b/extensions/data/default/jeeves/jeeves_listener.py @@ -95,6 +95,8 @@ def check_message_for_agent(self, message: dict) -> bool: """ payload_path = message.get(self.ct.PAYLOAD_DATA.EE_PAYLOAD_PATH, [None, None, None, None]) payload_signature = payload_path[2] if len(payload_path) >= 3 else None + explicit_signature = message.get(self.ct.SIGNATURE, None) + payload_signature = payload_signature or explicit_signature return payload_signature in JeevesCt.JEEVES_API_SIGNATURES diff --git a/extensions/serving/base/base_llm_serving.py b/extensions/serving/base/base_llm_serving.py index e7a04f19..a809ab39 100644 --- a/extensions/serving/base/base_llm_serving.py +++ b/extensions/serving/base/base_llm_serving.py @@ -124,6 +124,7 @@ "DEFAULT_TEMPERATURE" : 0.7, "DEFAULT_TOP_P" : 1, "DEFAULT_MAX_TOKENS" : 2048, + "DEFAULT_NUM_BEAMS" : 1, "SKIP_ERRORS" : True, "RELEVANT_SIGNATURES": None, "GENERATION_SEED": 42, # Seed for generation, can be set to None for random seed @@ -436,6 +437,8 @@ def _get_device_map(self): def check_relevant_input(self, input_dict: dict): inp_payload_path = input_dict.get(self.ct.PAYLOAD_DATA.EE_PAYLOAD_PATH, [None, None, None, None]) inp_signature = inp_payload_path[2] + explicit_signature = input_dict.get(self.ct.SIGNATURE, None) + inp_signature = inp_signature or explicit_signature normalized_signature = str(inp_signature).upper() if inp_signature is not None else None if normalized_signature not in self.get_relevant_signatures(): @@ -533,10 +536,10 @@ def _pre_process(self, inputs): } request_id = jeeves_content.get(LlmCT.REQUEST_ID, None) messages = jeeves_content.get(LlmCT.MESSAGES, []) - temperature = jeeves_content.get(LlmCT.TEMPERATURE) or self.cfg_default_temperature - top_p = jeeves_content.get(LlmCT.TOP_P) or self.cfg_default_top_p - max_tokens = jeeves_content.get(LlmCT.MAX_TOKENS) or self.cfg_default_max_tokens - repetition_penalty = jeeves_content.get("REPETITION_PENALTY", self.cfg_repetition_penalty) + temperature = jeeves_content.setdefault(LlmCT.TEMPERATURE, self.cfg_default_temperature) + top_p = jeeves_content.setdefault(LlmCT.TOP_P, self.cfg_default_top_p) + max_tokens = jeeves_content.setdefault(LlmCT.MAX_TOKENS, self.cfg_default_max_tokens) + repetition_penalty = jeeves_content.setdefault("REPETITION_PENALTY", self.cfg_repetition_penalty) request_context = jeeves_content.get(LlmCT.CONTEXT, None) valid_condition = jeeves_content.get(LlmCT.VALID_CONDITION, None) process_method = jeeves_content.get(LlmCT.PROCESS_METHOD, None) @@ -746,7 +749,10 @@ def extract_sql(self, text: str): if fence_match: return fence_match.group(1).strip() # exclude the back-ticks - # ── 3. Nothing found → give back the original string + # ── 3. Nothing found → give back the original string and maybe remove ``` + if text.strip().endswith("```"): + text = text.strip().strip("```").strip() + # endif text endswith ``` return text def remove_sql_comments(self, text: str): diff --git a/extensions/serving/default_inference/nlp/llama_sqlcoder.py b/extensions/serving/default_inference/nlp/llama_sqlcoder.py new file mode 100644 index 00000000..f77c3433 --- /dev/null +++ b/extensions/serving/default_inference/nlp/llama_sqlcoder.py @@ -0,0 +1,68 @@ +""" +Model from https://huggingface.co/defog/llama-3-sqlcoder-8b +""" + +from extensions.serving.base.base_llm_serving import BaseLlmServing as BaseServingProcess +from extensions.serving.mixins_llm.llm_utils import LlmCT + +__VER__ = '0.1.0.0' + +_CONFIG = { + **BaseServingProcess.CONFIG, + + "MODEL_NAME": "defog/llama-3-sqlcoder-8b", + + "PICKED_INPUT": "STRUCT_DATA", + "RUNS_ON_EMPTY_INPUT": False, + "DEFAULT_TEMPERATURE": 0.0, + + 'VALIDATION_RULES': { + **BaseServingProcess.CONFIG['VALIDATION_RULES'], + }, + +} + + +class LlamaSqlcoder(BaseServingProcess): + CONFIG = _CONFIG + + def generate_prompt(self, user_request, instructions=None): + res = f"""Generate a SQL DDL statement to answer this request: `{user_request}` +{instructions or ""} + +<|eot_id|><|start_header_id|>assistant<|end_header_id|> +The following SQL DDL statement best answers the request `{user_request}`: +```sql +""" + return res + + def _get_prompt_from_template(self, messages, context=None): + chat = self.preprocess_messages(messages, context) + prompt = None + if len(chat) > 0: + first_chat = chat[0] + instructions = None + if first_chat[LlmCT.ROLE_KEY] == LlmCT.SYSTEM_ROLE: + instructions = first_chat[LlmCT.DATA_KEY] + chat = chat[1:] + # endif system role + if len(chat) > 0: + last_message = chat[-1] + chat = chat[:-1] + prompt = self.generate_prompt(last_message[LlmCT.DATA_KEY], instructions) + else: + # If only system message exists, no processing is needed + chat = [first_chat] + # endif non-empty chat + # endif non-empty chat + if prompt is None: + date_string = self.datetime.now(self.timezone.utc).date().isoformat() + prompt = self.tokenizer.apply_chat_template( + chat, tokenize=False, + add_generation_prompt=self.cfg_add_generation_prompt, + # date_string=date_string, + ) + # endif prompt is None + self.P(f"Generated prompt:\n{prompt}") + return prompt + diff --git a/extensions/serving/default_inference/nlp/llama_sqlcoder_small.py b/extensions/serving/default_inference/nlp/llama_sqlcoder_small.py new file mode 100644 index 00000000..9908d9c8 --- /dev/null +++ b/extensions/serving/default_inference/nlp/llama_sqlcoder_small.py @@ -0,0 +1,46 @@ +""" +Model from https://huggingface.co/defog/llama-3-sqlcoder-8b +""" + +from extensions.serving.default_inference.nlp.llama_sqlcoder import LlamaSqlcoder as BaseServingProcess +from extensions.serving.mixins_llm.llm_utils import LlmCT + +__VER__ = '0.1.0.0' + +_CONFIG = { + **BaseServingProcess.CONFIG, + + "MODEL_NAME": "defog/sqlcoder-7b-2", + + "DEFAULT_NUM_BEAMS": 4, + + 'VALIDATION_RULES': { + **BaseServingProcess.CONFIG['VALIDATION_RULES'], + }, + +} + + +class LlamaSqlcoderSmall(BaseServingProcess): + CONFIG = _CONFIG + + def generate_prompt(self, user_request, instructions=None): + instructions_str = f"### Instructions \n{instructions}" if instructions else "" + res = f"""### Task +Generate a SQL DDL statement to answer [QUESTION]{user_request}[/QUESTION] +{instructions_str} + +### Answer +The following SQL DDL statement best answers the request `{user_request}`: + +### Database Schema +There is currently NO existing database schema. +You must design the necessary schema objects from scratch +using a single SQL DDL statement that satisfies the question. + +### Task +Given that there is currently no existing schema, here is the SQL DDL statement that satisfies [QUESTION]{user_request}[/QUESTION] +[SQL] +""" + return res + diff --git a/extensions/serving/mixins_llm/llm_model_mixin.py b/extensions/serving/mixins_llm/llm_model_mixin.py index 92626ca2..ddcc51ae 100644 --- a/extensions/serving/mixins_llm/llm_model_mixin.py +++ b/extensions/serving/mixins_llm/llm_model_mixin.py @@ -202,6 +202,11 @@ def get_model_predict_kwargs( **kwargs } + # This is done to avoid issues in case the default temperature of a model is 0.0 + # which could lead to issues when generating with sampling enabled. + res.setdefault("temperature", 1.0) + res.setdefault("num_beams", self.cfg_default_num_beams) + prompt_lens = attention_mask.sum(dim=1).tolist() temperatures = [pkwargs.get("temperature", self.cfg_default_temperature) for pkwargs in predict_kwargs_lst] top_ps = [pkwargs.get("top_p", self.cfg_default_top_p) for pkwargs in predict_kwargs_lst] @@ -210,13 +215,22 @@ def get_model_predict_kwargs( max_new_tokens = [pkwargs.get("max_new_tokens", self.cfg_default_max_tokens) for pkwargs in predict_kwargs_lst] eos_token_id = self.tokenizer.eos_token_id - - logits_processors = LogitsProcessorList([ - PerSampleTemperature(temperatures), + max_temperature = max(temperatures) + lst_logits_processors = [ PerSampleTopP(top_ps), PerSampleRepetitionPenalty(penalties), PerSampleMaxLength(prompt_lens, max_new_tokens, eos_token_id), - ]) + ] + if max_temperature > 0.0: + lst_logits_processors.append( + PerSampleTemperature(temperatures) + ) + else: + res["temperature"] = 0.0 + res["do_sample"] = False + # endif max_temperature > 0.0 + + logits_processors = LogitsProcessorList(lst_logits_processors) max_ceiling = max(pl + mnt for pl, mnt in zip(prompt_lens, max_new_tokens)) res["max_length"] = max_ceiling diff --git a/extensions/serving/mixins_llm/llm_tokenizer_mixin.py b/extensions/serving/mixins_llm/llm_tokenizer_mixin.py index 4a592d63..c3eb1ddd 100644 --- a/extensions/serving/mixins_llm/llm_tokenizer_mixin.py +++ b/extensions/serving/mixins_llm/llm_tokenizer_mixin.py @@ -37,7 +37,7 @@ def add_context_to_request(self, request, context: list): f"---" ) - def _get_prompt_from_template(self, messages, context=None): + def preprocess_messages(self, messages, context=None): """ Uses Jinja template to generate a prompt. @@ -107,13 +107,38 @@ def _get_prompt_from_template(self, messages, context=None): # endif non-empty chat # endif context provided # The context feature is disabled until further improvements are made. + return chat + + def _get_prompt_from_template(self, messages, context=None): + """ + Uses Jinja template to generate a prompt. + + Parameters + ---------- + messages : list[dict] + List of dictionaries, where each dictionary represents a message in the conversation. + Each dictionary should have the keys 'role' and 'content'. + The 'role' key should be one of 'user', 'assistant', or 'system'. + context : list or str, optional + the context for the prompt - CURRENTLY DISABLED + + Returns + ------- + str + full prompt + Raises + ------ + ValueError + _description_ + """ + chat = self.preprocess_messages(messages, context) self.P(f"Processing chat:\n{chat}") date_string = self.datetime.now(self.timezone.utc).date().isoformat() from_template = self.tokenizer.apply_chat_template( chat, tokenize=False, add_generation_prompt=self.cfg_add_generation_prompt, - date_string=date_string + # date_string=date_string ) return from_template diff --git a/extensions/serving/mixins_llm/llm_utils.py b/extensions/serving/mixins_llm/llm_utils.py index e84cec07..2a833740 100644 --- a/extensions/serving/mixins_llm/llm_utils.py +++ b/extensions/serving/mixins_llm/llm_utils.py @@ -107,8 +107,48 @@ class LlmCT: """LOGITS PROCESSOR SECTION""" if True: + # 0. Base class for per-sample logits processors + class BasePerSampleLogitsProcessor(LogitsProcessor): + """ + Helper base class for per-sample logits processors that must also work + when beam search is enabled (num_beams > 1). + + It expands a per-sample vector of shape (B,) to match the *effective* + batch dimension during generation, which is batch_beam_size = B * num_beams. + """ + + @staticmethod + def _expand_vector( + vec: th.Tensor, + batch_beam_size: int, + device: th.device, + name: str, + ) -> th.Tensor: + """ + Expand a (B,) tensor `vec` to (batch_beam_size,) by repeating across beams + when batch_beam_size != B. + + If batch_beam_size == B, the vector is returned as-is. + """ + vec = vec.to(device) + base_B = vec.size(0) + + if batch_beam_size == base_B: + # No beams or already expanded. + return vec + + if batch_beam_size % base_B != 0: + raise ValueError( + f"{name}: batch_beam_size={batch_beam_size} is not divisible " + f"by per-sample size {base_B}." + ) + + num_beams = batch_beam_size // base_B + # (B,) -> (B, num_beams) -> (B * num_beams,) + return vec.unsqueeze(1).expand(-1, num_beams).reshape(-1) + # 1. Per-row Temperature with greedy fallback - class PerSampleTemperature(LogitsProcessor): + class PerSampleTemperature(BasePerSampleLogitsProcessor): """ Vectorised temperature scaling that *also* supports T == 0 -> greedy. - temps : list / 1-D tensor with length == batch_size. @@ -134,20 +174,43 @@ def __init__(self, temps: Sequence[float]): safe_t[self.greedy_mask] = 1.0 # avoid divide-by-zero self.inv_t = 1.0 / safe_t # store reciprocal for faster mul self.has_greedy = self.greedy_mask.any() + return def __call__(self, input_ids: th.Tensor, scores: th.Tensor) -> th.Tensor: + device = scores.device + batch_beam_size, _ = scores.shape + + # Expand per-sample 1/T to per-(sample,beam) + eff_inv_t = self._expand_vector(self.inv_t, batch_beam_size, device, + "PerSampleTemperature.inv_t") + # --- 1 Scale logits (scores *= 1/T) --------------------------- - scores.mul_(self.inv_t.to(scores.device).unsqueeze(-1)) + scores.mul_(eff_inv_t.unsqueeze(-1)) # --- 2 Force arg-max for rows with T == 0 --------------------- if self.has_greedy: # host flag -> no GPU sync cost - gmask = self.greedy_mask.to(scores.device) # (B,) - row_idx = th.nonzero(gmask, as_tuple=True)[0] # rows that are greedy - col_idx = scores[gmask].argmax(dim=-1) # winning token per row - - # ❶ Set full row to −inf in bulk, ❷ restore the winner to 0 - scores[gmask] = -float("inf") # boolean-mask assignment is fused. - scores[row_idx, col_idx] = 0.0 + eff_gmask = self._expand_vector( + self.greedy_mask, batch_beam_size, device, "PerSampleTemperature.greedy_mask" + ) # (B * num_beams,) + + row_idx = th.nonzero(eff_gmask, as_tuple=True)[0] # rows that are greedy + col_idx = scores[eff_gmask].argmax(dim=-1) # winning token per row + + scores[eff_gmask] = -float("inf") # bulk-mask rows + scores[row_idx, col_idx] = 0.0 # restore winners + + # # --- 1 Scale logits (scores *= 1/T) --------------------------- + # scores.mul_(self.inv_t.to(scores.device).unsqueeze(-1)) + # + # # --- 2 Force arg-max for rows with T == 0 --------------------- + # if self.has_greedy: # host flag -> no GPU sync cost + # gmask = self.greedy_mask.to(scores.device) # (B,) + # row_idx = th.nonzero(gmask, as_tuple=True)[0] # rows that are greedy + # col_idx = scores[gmask].argmax(dim=-1) # winning token per row + # + # # ❶ Set full row to −inf in bulk, ❷ restore the winner to 0 + # scores[gmask] = -float("inf") # boolean-mask assignment is fused. + # scores[row_idx, col_idx] = 0.0 return scores @@ -155,7 +218,7 @@ def __repr__(self): return f"{self.__class__.__name__}(temps={self.inv_t.tolist()})" # 2. Per-row Top-p (nucleus) sampling - class PerSampleTopP(LogitsProcessor): + class PerSampleTopP(BasePerSampleLogitsProcessor): """ Vectorised nucleus-filtering with a *different* p for every row. For each sequence we keep the smallest set of tokens whose cum-prob >= p, @@ -168,14 +231,23 @@ def __init__(self, top_ps: Sequence[float], min_tokens_to_keep: int = 1): raise ValueError("top_p must be in (0, 1].") self.ps = ps self.min_keep = min_tokens_to_keep + return def __call__(self, input_ids: th.Tensor, logits: th.Tensor) -> th.Tensor: + device = logits.device + batch_beam_size, _ = logits.shape + + # Expand per-sample top_p to per-(sample,beam) + eff_ps = self._expand_vector( + self.ps, batch_beam_size, device, "PerSampleTopP.ps" + ) + # 1 Convert logits->probs; sort descending to get cumsum easily probs, idx = logits.softmax(dim=-1).sort(dim=-1, descending=True) cumprobs = probs.cumsum(dim=-1) # 2 For each row, mask tokens once cum prob exceeds its own p - cut_mask = cumprobs > self.ps.to(logits.device).unsqueeze(-1) + cut_mask = cumprobs > eff_ps.unsqueeze(-1) cut_mask[..., : self.min_keep] = False # guarantee >=min_keep tokens # 3 Translate mask back to original vocab order in-place @@ -189,7 +261,7 @@ def __repr__(self): return f"{self.__class__.__name__}(top_ps={self.ps.tolist()}, min_tokens_to_keep={self.min_keep})" # 3. Per-row Repetition Penalty - class PerSampleRepetitionPenalty(LogitsProcessor): + class PerSampleRepetitionPenalty(BasePerSampleLogitsProcessor): """ Implements the algorithm from HF's RepetitionPenaltyLogitsProcessor but with a vector of penalties, one per sequence, and **no Python loop**. @@ -207,17 +279,23 @@ def __init__(self, penalties: Sequence[float]): if (p <= 0).any(): raise ValueError("repetition_penalty must be > 0.") self.pen = p + return def __call__(self, input_ids: th.Tensor, logits: th.Tensor) -> th.Tensor: - B, V = logits.shape + B_eff, V = logits.shape device = logits.device + # Expand per-sample penalty to per-(sample,beam) + eff_pen = self._expand_vector( + self.pen, B_eff, device, "PerSampleRepetitionPenalty.pen" + ) + # 1 Build (B,V) mask of tokens present in each prefix - seen_tok = th.zeros((B, V), dtype=th.bool, device=device) + seen_tok = th.zeros((B_eff, V), dtype=th.bool, device=device) seen_tok.scatter_(1, input_ids, True) # O(B·seq_len) write # 2 Broadcast row-wise penalty factors - pen = self.pen.to(device).unsqueeze(-1) # (B,1) -> (B,V) via broadcast + pen = eff_pen.unsqueeze(-1) # (B,1) -> (B,V) via broadcast # 3 Apply formula in one fused where # > if l > 0: l = l / p @@ -234,7 +312,7 @@ def __repr__(self): return f"{self.__class__.__name__}(penalties={self.pen.tolist()})" # 4. Per-row Max-new-tokens gate - class PerSampleMaxLength(LogitsProcessor): + class PerSampleMaxLength(BasePerSampleLogitsProcessor): """ Soft EOS gate used instead of the global `max_new_tokens` arg. As soon as *any* row reaches its personal length budget, we force @@ -253,14 +331,25 @@ def __init__( self.target_len = (th.as_tensor(prompt_lens) + th.as_tensor(max_new_tokens)) self.eos_id = eos_token_id + return def __call__(self, input_ids: th.Tensor, logits: th.Tensor) -> th.Tensor: + device = input_ids.device + batch_beam_size = input_ids.size(0) cur_len = input_ids.size(1) - done_mask = cur_len >= self.target_len.to(input_ids.device) # (B,) + + # Expand per-sample target_len to per-(sample,beam) + eff_target_len = self._expand_vector( + self.target_len, batch_beam_size, device, "PerSampleMaxLength.target_len" + ) + + # Now eff_target_len shape matches batch_beam_size + done_mask = cur_len >= eff_target_len # (batch_beam_size,) # Bulk-mask done rows; boolean index is cheap/no-op when all-False. logits[done_mask] = -float("inf") logits[done_mask, self.eos_id] = 0.0 + return logits def __repr__(self): From 976d6a88f92fd4022a0adfe12517dde8bff5b30d Mon Sep 17 00:00:00 2001 From: Cristi Bleotiu Date: Thu, 27 Nov 2025 11:32:28 +0200 Subject: [PATCH 09/11] chore: inc ver --- ver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ver.py b/ver.py index 4e624df8..3b8629ec 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.900' +__VER__ = '2.9.910' From d8405be9ab90c8681b8ee7dab85b1fb7759b6d60 Mon Sep 17 00:00:00 2001 From: Alessandro <37877991+aledefra@users.noreply.github.com> Date: Thu, 27 Nov 2025 14:31:59 +0100 Subject: [PATCH 10/11] feat: tunnels manager updates for services (#316) --- .../business/tunnels/tunnels_manager.py | 41 +++++++++++++++---- ver.py | 2 +- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/extensions/business/tunnels/tunnels_manager.py b/extensions/business/tunnels/tunnels_manager.py index c6c5e6ac..684c602b 100644 --- a/extensions/business/tunnels/tunnels_manager.py +++ b/extensions/business/tunnels/tunnels_manager.py @@ -1,8 +1,9 @@ from naeural_core.business.default.web_app.supervisor_fast_api_web_app import SupervisorFastApiWebApp as BasePlugin -__VER__ = '0.0.2' +__VER__ = '0.1.0' MESSAGE_PREFIX = "Please sign this message to manage your tunnels: " +MESSAGE_PREFIX_DEEPLOY = "Please sign this message for Deeploy: " _CONFIG = { **BasePlugin.CONFIG, @@ -65,12 +66,21 @@ def get_secrets(self, payload: dict): Get Cloudflare secrets for the sender address. """ self._verify_nonce(payload['nonce']) - sender = self.bc.eth_verify_payload_signature( - payload=payload, - message_prefix=MESSAGE_PREFIX, - no_hash=True, - indent=1, - ) + sender = None + signature_errors = [] + for prefix in (MESSAGE_PREFIX, MESSAGE_PREFIX_DEEPLOY): + try: + sender = self.bc.eth_verify_payload_signature( + payload=payload, + message_prefix=prefix, + no_hash=True, + indent=1, + ) + break + except Exception as exc: + signature_errors.append(str(exc)) + if sender is None: + raise Exception(f"Signature verification failed for provided payload: {signature_errors}") secrets = self.chainstore_hget(hkey="tunnels_manager_secrets", key=sender) # TODO we should add a CSP password to be used as token in cstore if secrets is None: @@ -110,11 +120,12 @@ def check_secrets_exist(self, csp_address: str): } @BasePlugin.endpoint(method="post") - def new_tunnel(self, alias: str, cloudflare_account_id: str, cloudflare_zone_id: str, cloudflare_api_key: str, cloudflare_domain: str): + def new_tunnel(self, alias: str, cloudflare_account_id: str, cloudflare_zone_id: str, cloudflare_api_key: str, cloudflare_domain: str, service_name: str | None = None,): """ Create a new Cloudflare tunnel. """ - new_id = self.uuid() + new_uuid = self.uuid() + new_id = f"{service_name}-{new_uuid}" if service_name is not None else new_uuid url = f"{self.cfg_base_cloudflare_url}/client/v4/accounts/{cloudflare_account_id}/cfd_tunnel" headers = { "Authorization": f"Bearer {cloudflare_api_key}" @@ -182,6 +193,18 @@ def get_tunnel(self, tunnel_id: str, cloudflare_account_id: str, cloudflare_api_ raise Exception("Error fetching tunnel: " + str(response['errors'])) return response['result'] + @BasePlugin.endpoint(method="get") + def get_tunnel_by_token(self, tunnel_token: str, cloudflare_account_id: str, cloudflare_api_key: str): + """ + Get tunnel details using its tunnel token. + """ + tunnels = self.get_tunnels(cloudflare_account_id, cloudflare_api_key) + for tunnel in tunnels: + metadata = tunnel.get('metadata', {}) + if metadata.get('tunnel_token') == tunnel_token: + return tunnel + raise Exception("Tunnel not found for provided token.") + @BasePlugin.endpoint(method="delete") def delete_tunnel(self, tunnel_id: str, cloudflare_account_id: str, cloudflare_zone_id: str, cloudflare_api_key: str): """ diff --git a/ver.py b/ver.py index 3b8629ec..b6b45fb9 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.910' +__VER__ = '2.9.911' From 6f26b89f94566baa7fbd27d74a62fa05caddb070 Mon Sep 17 00:00:00 2001 From: Cristi Bleotiu Date: Thu, 27 Nov 2025 15:54:12 +0200 Subject: [PATCH 11/11] chore: inc ver --- ver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ver.py b/ver.py index b6b45fb9..323e924d 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.9.911' +__VER__ = '2.9.920'