diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..71d6f2f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,99 @@ +name: CI + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + test: + name: Test on Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.11', '3.12'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run tests + run: | + pytest tests/ -v --cov=src/debugai --cov-report=xml --cov-report=term + + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + name: codecov-${{ matrix.os }}-py${{ matrix.python-version }} + + lint: + name: Lint and Format Check + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install ruff black mypy + + - name: Run Ruff + run: ruff check . + + - name: Run Black + run: black --check . + + - name: Run MyPy + run: mypy src/debugai --ignore-missing-imports + + stress-test: + name: Run Stress Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -e ".[dev]" + + - name: Run stress tests + run: | + python tests/stress_test.py + + - name: Upload stress test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: stress-test-results + path: | + stress-test-*.log + stress-test-*.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af6d74b --- /dev/null +++ b/.gitignore @@ -0,0 +1,65 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual Environment +venv/ +ENV/ +env/ +.venv + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.hypothesis/ + +# Environment +.env +.env.local +.env.*.local + +# Logs +*.log +logs/ + +# Database +*.db +*.sqlite +*.sqlite3 + +# OS +.DS_Store +Thumbs.db + +# DebugAI specific +.debugai/ +stress-test-*.log +stress-test-*.json diff --git a/debugai/.github/workflows/ci.yml b/debugai/.github/workflows/ci.yml index 8a6300a..71d6f2f 100644 --- a/debugai/.github/workflows/ci.yml +++ b/debugai/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Run tests run: | - pytest tests/ -v --cov=debugai --cov-report=xml --cov-report=term + pytest tests/ -v --cov=src/debugai --cov-report=xml --cov-report=term - name: Upload coverage uses: codecov/codecov-action@v3 @@ -90,7 +90,7 @@ jobs: python tests/stress_test.py - name: Upload stress test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: stress-test-results diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9735d81 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +""" +Pytest configuration and fixtures +""" +import sys +from pathlib import Path + +# Add src to path for imports +project_root = Path(__file__).parent.parent +src_path = project_root / "src" +sys.path.insert(0, str(src_path)) diff --git a/tests/stress_test.py b/tests/stress_test.py new file mode 100644 index 0000000..097da56 --- /dev/null +++ b/tests/stress_test.py @@ -0,0 +1,444 @@ +""" +Stress Test Suite for DebugAI +Tests performance under high load and concurrent operations +""" + +import sys +import time +import random +import threading +from pathlib import Path +from datetime import datetime, timedelta +from typing import List, Dict, Any +import tempfile +import shutil + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from debugai.ingestion.parser import LogParser +from debugai.core.analyzer import LogAnalyzer +from debugai.storage.database import Database + + +class StressTestRunner: + """Run stress tests for DebugAI components""" + + def __init__(self): + self.results = {} + self.temp_dir = None + + def setup(self): + """Setup test environment""" + print("๐Ÿ”ง Setting up test environment...") + self.temp_dir = tempfile.mkdtemp(prefix="debugai_stress_") + print(f" Temp directory: {self.temp_dir}") + + def teardown(self): + """Cleanup test environment""" + print("\n๐Ÿงน Cleaning up...") + if self.temp_dir and Path(self.temp_dir).exists(): + try: + # Give time for any file handles to close + time.sleep(0.2) + shutil.rmtree(self.temp_dir) + print(" โœ“ Cleanup complete") + except Exception as e: + print(f" โš  Warning: Could not remove temp directory: {e}") + + def generate_log_entry(self, index: int, service: str = "api") -> Dict[str, Any]: + """Generate realistic log entry as dictionary""" + levels = ["INFO", "WARN", "ERROR", "DEBUG"] + errors = [ + "Connection timeout to database", + "NullPointerException in UserService", + "Failed to authenticate user", + "Out of memory error", + "Database query timeout", + "Network connection refused", + "Invalid API key provided", + "Rate limit exceeded", + ] + + level = random.choice(levels) + timestamp = datetime.now() - timedelta(minutes=random.randint(0, 60)) + + if level == "ERROR": + message = random.choice(errors) + else: + message = f"Processing request #{index}" + + raw_log = f"{timestamp.isoformat()} [{level}] [{service}] {message}" + + return { + "raw": raw_log, + "service": service, + "timestamp": timestamp.isoformat() + } + + def test_parser_throughput(self, log_count: int = 10000): + """Test log parser throughput""" + print(f"\n๐Ÿ“Š Testing Parser Throughput ({log_count:,} logs)...") + + # Generate logs + print(" Generating logs...") + logs = [self.generate_log_entry(i) for i in range(log_count)] + + # Parse logs + parser = LogParser() + print(" Parsing logs...") + start = time.time() + + parsed = [] + errors = 0 + for log in logs: + try: + result = parser.parse(log) + if result: + parsed.append(result) + except Exception as e: + errors += 1 + + duration = time.time() - start + if duration == 0: + duration = 0.001 # Prevent division by zero + throughput = len(parsed) / duration + + print(f" โœ“ Parsed {len(parsed):,}/{log_count:,} logs in {duration:.2f}s") + print(f" โœ“ Throughput: {throughput:,.0f} logs/sec") + if errors > 0: + print(f" โš  Parse errors: {errors}") + + self.results['parser_throughput'] = throughput + self.results['parsed_count'] = len(parsed) + return throughput + + def test_analyzer_performance(self, log_count: int = 5000): + """Test analyzer performance with large datasets""" + print(f"\n๐Ÿ“Š Testing Analyzer Performance ({log_count:,} logs)...") + + # Generate and parse logs + print(" Generating logs...") + services = ["api", "db", "redis", "cache"] + logs = [] + + for i in range(log_count): + service = random.choice(services) + log_data = self.generate_log_entry(i, service) + + # Create simpler log dict for analyzer + logs.append({ + "timestamp": log_data["timestamp"], + "level": random.choice(["INFO", "WARN", "ERROR", "DEBUG"]), + "service": service, + "message": log_data["raw"], + "raw": log_data["raw"] + }) + + # Analyze logs + analyzer = LogAnalyzer() + print(" Analyzing logs...") + start = time.time() + + try: + results = analyzer.analyze(logs) + duration = time.time() - start + + patterns = results.get('patterns', []) + error_types = results.get('error_types', {}) + + print(f" โœ“ Analyzed {len(logs):,} logs in {duration:.2f}s") + print(f" โœ“ Found {len(patterns)} patterns") + print(f" โœ“ Found {len(error_types)} error types") + + self.results['analyzer_duration'] = duration + self.results['patterns_found'] = len(patterns) + except Exception as e: + print(f" โš  Analyzer error: {e}") + self.results['analyzer_duration'] = 0 + + def test_database_operations(self, operation_count: int = 500): + """Test database write/read performance""" + print(f"\n๐Ÿ“Š Testing Database Operations ({operation_count:,} ops)...") + + db_path = Path(self.temp_dir) / "test.db" + + try: + db = Database(str(db_path)) + parser = LogParser() + + # Test writes + print(" Testing writes...") + stored_count = 0 + start = time.time() + + for i in range(operation_count): + log_entry = self.generate_log_entry(i) + try: + parsed = parser.parse(log_entry) + if parsed: + db.store_log(parsed, source_name=f"stress-test-{random.randint(1, 3)}") + stored_count += 1 + except Exception as e: + pass + + write_duration = time.time() - start + if write_duration == 0: + write_duration = 0.001 + write_throughput = stored_count / write_duration + + print(f" โœ“ Writes: {stored_count:,}/{operation_count:,} in {write_duration:.2f}s ({write_throughput:,.0f} ops/sec)") + + # Test reads + if stored_count > 0: + print(" Testing reads...") + start = time.time() + read_count = min(50, stored_count) + for _ in range(read_count): + try: + results = db.get_logs(limit=10) + except: + pass + read_duration = time.time() - start + + print(f" โœ“ Reads: {read_count} queries in {read_duration:.2f}s") + self.results['db_read_duration'] = read_duration + + self.results['db_write_throughput'] = write_throughput + self.results['stored_count'] = stored_count + + # Close database + try: + db.close() + except: + pass + + # Wait for cleanup + time.sleep(0.1) + + except Exception as e: + print(f" โš  Database test error: {e}") + + def test_concurrent_parsing(self, thread_count: int = 5, logs_per_thread: int = 500): + """Test concurrent log parsing""" + print(f"\n๐Ÿ“Š Testing Concurrent Parsing ({thread_count} threads, {logs_per_thread:,} logs each)...") + + results = [] + errors = [] + lock = threading.Lock() + + def parse_logs(thread_id: int): + """Parse logs in thread""" + parser = LogParser() + local_results = [] + local_errors = [] + + for i in range(logs_per_thread): + log_entry = self.generate_log_entry(i, f"service-{thread_id}") + try: + parsed = parser.parse(log_entry) + if parsed: + local_results.append(parsed) + except Exception as e: + local_errors.append(str(e)) + + with lock: + results.extend(local_results) + errors.extend(local_errors) + + # Create and start threads + threads = [] + start = time.time() + + for i in range(thread_count): + thread = threading.Thread(target=parse_logs, args=(i,)) + threads.append(thread) + thread.start() + + # Wait for completion + for thread in threads: + thread.join() + + duration = time.time() - start + total_expected = thread_count * logs_per_thread + if duration == 0: + duration = 0.001 + throughput = len(results) / duration + + print(f" โœ“ Parsed {len(results):,}/{total_expected:,} logs in {duration:.2f}s") + print(f" โœ“ Throughput: {throughput:,.0f} logs/sec") + if errors: + print(f" โš  Errors: {len(errors)}") + + self.results['concurrent_throughput'] = throughput + self.results['concurrent_parsed'] = len(results) + + def test_large_batch_processing(self, log_count: int = 20000): + """Test processing large batches of logs""" + print(f"\n๐Ÿ“Š Testing Large Batch Processing ({log_count:,} logs)...") + + print(" Generating logs...") + logs = [] + for i in range(log_count): + logs.append(self.generate_log_entry(i, f"service-{i % 5}")) + + print(" Processing batch...") + parser = LogParser() + start = time.time() + + parsed = [] + for log in logs: + try: + result = parser.parse(log) + if result: + parsed.append(result.to_dict()) + except: + pass + + duration = time.time() - start + if duration == 0: + duration = 0.001 + + # Calculate memory usage estimate + import sys + if parsed: + memory_mb = sys.getsizeof(parsed) / (1024 * 1024) + per_log_kb = (memory_mb / len(parsed)) * 1024 + else: + memory_mb = 0 + per_log_kb = 0 + + print(f" โœ“ Processed {len(parsed):,} logs in {duration:.2f}s") + print(f" โœ“ Throughput: {len(parsed)/duration:,.0f} logs/sec") + print(f" โœ“ Memory usage: ~{memory_mb:.2f} MB") + if parsed: + print(f" โœ“ Per log: ~{per_log_kb:.2f} KB") + + self.results['batch_throughput'] = len(parsed) / duration + self.results['memory_mb'] = memory_mb + + def test_error_detection(self, log_count: int = 3000): + """Test error detection and categorization""" + print(f"\n๐Ÿ“Š Testing Error Detection ({log_count:,} logs)...") + + # Generate logs with specific error patterns + print(" Generating logs with errors...") + logs = [] + error_count = 0 + + for i in range(log_count): + # 30% errors + if random.random() < 0.3: + level = "ERROR" + error_count += 1 + else: + level = random.choice(["INFO", "WARN", "DEBUG"]) + + service = random.choice(["api", "db", "redis"]) + log_entry = self.generate_log_entry(i, service) + + logs.append({ + "timestamp": log_entry["timestamp"], + "level": level, + "service": service, + "message": log_entry["raw"], + "raw": log_entry["raw"] + }) + + print(f" Generated {error_count} error logs") + + # Analyze + analyzer = LogAnalyzer() + start = time.time() + + try: + results = analyzer.analyze(logs) + duration = time.time() - start + + error_types = results.get('error_types', {}) + anomalies = results.get('anomalies', []) + + print(f" โœ“ Analyzed in {duration:.2f}s") + print(f" โœ“ Error types detected: {len(error_types)}") + print(f" โœ“ Anomalies found: {len(anomalies)}") + + self.results['error_detection_duration'] = duration + except Exception as e: + print(f" โš  Error detection failed: {e}") + + def print_summary(self): + """Print test summary""" + print("\n" + "="*60) + print("๐Ÿ“ˆ STRESS TEST SUMMARY") + print("="*60) + + if 'parser_throughput' in self.results: + print(f"Parser Throughput: {self.results['parser_throughput']:>15,.0f} logs/sec") + print(f" Parsed Successfully: {self.results.get('parsed_count', 0):>15,} logs") + + if 'analyzer_duration' in self.results and self.results['analyzer_duration'] > 0: + print(f"\nAnalyzer Performance:") + print(f" Duration: {self.results['analyzer_duration']:>15.2f} seconds") + print(f" Patterns Found: {self.results.get('patterns_found', 0):>15}") + + if 'db_write_throughput' in self.results: + print(f"\nDatabase Performance:") + print(f" Write Throughput: {self.results['db_write_throughput']:>15,.0f} ops/sec") + print(f" Records Stored: {self.results.get('stored_count', 0):>15,}") + if 'db_read_duration' in self.results: + print(f" Read Duration: {self.results['db_read_duration']:>15.2f} seconds") + + if 'concurrent_throughput' in self.results: + print(f"\nConcurrent Processing:") + print(f" Throughput: {self.results['concurrent_throughput']:>15,.0f} logs/sec") + print(f" Parsed Successfully: {self.results.get('concurrent_parsed', 0):>15,}") + + if 'batch_throughput' in self.results: + print(f"\nBatch Processing:") + print(f" Throughput: {self.results['batch_throughput']:>15,.0f} logs/sec") + print(f" Memory Usage: {self.results.get('memory_mb', 0):>15.2f} MB") + + if 'error_detection_duration' in self.results: + print(f"\nError Detection:") + print(f" Duration: {self.results['error_detection_duration']:>15.2f} seconds") + + print("="*60) + print("โœ… All stress tests completed!") + print("="*60) + + +def main(): + """Run all stress tests""" + print("โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—") + print("โ•‘ DebugAI Stress Test Suite โ•‘") + print("โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•") + print() + + runner = StressTestRunner() + + try: + runner.setup() + + # Run all tests + runner.test_parser_throughput(log_count=10000) + runner.test_analyzer_performance(log_count=5000) + runner.test_database_operations(operation_count=500) + runner.test_concurrent_parsing(thread_count=5, logs_per_thread=500) + runner.test_large_batch_processing(log_count=20000) + runner.test_error_detection(log_count=3000) + + # Print summary + runner.print_summary() + + except KeyboardInterrupt: + print("\n\nโš  Tests interrupted by user") + except Exception as e: + print(f"\nโŒ Error during stress test: {e}") + import traceback + traceback.print_exc() + finally: + runner.teardown() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..845dcd4 --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,58 @@ +""" +Basic unit tests for DebugAI +""" +import pytest +from pathlib import Path + + +def test_package_import(): + """Test that the debugai package can be imported""" + try: + import debugai + assert debugai is not None + except ImportError as e: + pytest.fail(f"Failed to import debugai: {e}") + + +def test_parser_import(): + """Test that the parser module can be imported""" + try: + from debugai.ingestion.parser import LogParser + assert LogParser is not None + except ImportError as e: + pytest.fail(f"Failed to import LogParser: {e}") + + +def test_analyzer_import(): + """Test that the analyzer module can be imported""" + try: + from debugai.core.analyzer import LogAnalyzer + assert LogAnalyzer is not None + except ImportError as e: + pytest.fail(f"Failed to import LogAnalyzer: {e}") + + +def test_database_import(): + """Test that the database module can be imported""" + try: + from debugai.storage.database import Database + assert Database is not None + except ImportError as e: + pytest.fail(f"Failed to import Database: {e}") + + +def test_cli_import(): + """Test that the CLI module can be imported""" + try: + from debugai.cli.main import app + assert app is not None + except ImportError as e: + pytest.fail(f"Failed to import CLI app: {e}") + + +def test_version(): + """Test that version is defined""" + import debugai + assert hasattr(debugai, '__version__') + assert isinstance(debugai.__version__, str) + assert len(debugai.__version__) > 0