diff --git a/.gitignore b/.gitignore index 265d138..9fb33f6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ shadow_deepscan_report/ __pycache__/ *.pcap *.pcapng +ctf_config.yaml +flag_submissions.log +ctf_analysis.md +auto_solver_results.txt +suggested_tools.txt diff --git a/IMPLEMENTATION_SUMMARY.md b/IMPLEMENTATION_SUMMARY.md new file mode 100644 index 0000000..34cd72f --- /dev/null +++ b/IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,209 @@ +# CTF Features Implementation Summary + +This document summarizes the comprehensive CTF-specific features added to ShadowParse. + +## Overview + +ShadowParse has been enhanced with a complete CTF automation toolkit that includes auto-solving capabilities, intelligent challenge categorization, progressive hints, and flag submission to popular CTF platforms. + +## New Modules + +### 1. ctf_solvers.py (15 KB) +**Auto-Solver Library** - Implements automated solving techniques: +- **ROT Ciphers**: Tests all ROT1-25 variations with readability scoring +- **XOR Bruteforce**: Single-byte and multi-byte XOR key search +- **Substitution Ciphers**: Frequency analysis-based solving +- **CyberChef Magic**: Common encoding chains (Base64→Gunzip→Hex, Hex→Base64, URL→Base64) +- **QR Code/Barcode Detection**: Automatic extraction and decoding from images +- **Shellcode Detection**: Pattern-based detection of shellcode +- **Binary Disassembly**: Supports i386, amd64, arm, aarch64, mips architectures + +**Key Features**: +- Confidence scoring for all results +- Readability analysis for decoded text +- Safe execution without arbitrary code execution +- Graceful handling when dependencies are missing + +### 2. ctf_categorizer.py (9.5 KB) +**Challenge Categorization System** - Intelligently categorizes challenges: + +**Supported Categories**: +- **Cryptography**: High entropy, encoded data, cipher patterns +- **Forensics**: File carving, hidden data, metadata analysis +- **Steganography**: Image files, audio files, LSB patterns +- **Web Exploitation**: HTTP traffic, SQL injection, XSS patterns +- **Network Analysis**: DNS tunneling, covert channels, protocol abuse +- **Reverse Engineering**: Binary extraction, shellcode, obfuscated code + +**Scoring System**: +- Weighted indicators for each category +- Normalized confidence scores (0-1) +- Primary and secondary category detection +- Threshold-based secondary category reporting (>30% confidence) + +### 3. hint_engine.py (14 KB) +**Intelligent Hint System** - Provides context-aware hints: + +**Hint Categories**: +- High entropy data handling +- Encoding/decoding strategies +- DNS analysis techniques +- File analysis methods +- Network traffic patterns +- Web exploitation approaches +- Cryptographic techniques +- Flag finding strategies + +**Features**: +- Progressive difficulty levels (basic, intermediate, advanced) +- Context-aware hint selection +- Customizable hints via hints.json +- Tool recommendations per category +- Actionable advice with specific tool names + +### 4. ctf_submission.py (11 KB) +**Flag Submission Integration** - Automated flag submission: + +**Supported Platforms**: +- **CTFd**: Full API integration with challenge ID support +- **HackTheBox**: Machine flag submission (user.txt and root.txt) +- **Webhook**: Generic webhook for custom integrations + +**Features**: +- Configurable via YAML file +- Confirmation prompts (default: enabled) +- Auto-submit mode (opt-in) +- Submission logging +- Error handling and timeouts +- Custom headers for webhooks + +## Enhanced Main Engine (shadowparse.py) + +### New CLI Arguments +```bash +--ctf-mode # Enable CTF-specific features +--auto-solve # Run auto-solver tools +--submit-flags # Auto-submit found flags (requires --config) +--hints # Show progressive hints +--config # Path to CTF configuration file +``` + +### Integration Points +1. **Initialization**: CTF modules loaded optionally with graceful degradation +2. **Analysis Phase**: Auto-solvers run on high-entropy data and extracted files +3. **Categorization**: Challenge categorized based on PCAP analysis +4. **Hint Generation**: Context-aware hints generated from findings +5. **Flag Submission**: Optional automatic flag submission + +### New Output Files (CTF Mode) +- `ctf_analysis.md`: Challenge category, hints, tool recommendations +- `auto_solver_results.txt`: Results from automated solving attempts +- `suggested_tools.txt`: Category-specific tool recommendations +- `flag_submissions.log`: Log of flag submission attempts + +## Configuration + +### ctf_config.yaml Template +```yaml +ctf_platform: + enabled: false + platform: "ctfd" # or "htb", "webhook" + url: "https://ctf.example.com" + api_token: "your-token-here" + auto_submit: false +``` + +## Usage Examples + +### Basic CTF Mode +```bash +python shadowparse.py -f evidence.pcap --ctf-mode --hints +``` + +### Full CTF Mode with Auto-Solver +```bash +python shadowparse.py -f evidence.pcap --ctf-mode --auto-solve --hints +``` + +### With Flag Submission +```bash +python shadowparse.py -f evidence.pcap --ctf-mode --auto-solve --submit-flags --config ctf_config.yaml +``` + +## Dependencies + +### Required (existing) +- scapy +- pandas +- rich +- chardet + +### Optional (CTF features) +- pwntools>=4.11.0 (binary analysis, disassembly) +- qrcode>=7.4.2 (QR code generation) +- pillow>=10.0.0 (image processing) +- pyzbar>=0.1.9 (QR/barcode decoding) +- python-magic>=0.4.27 (file type detection) +- requests>=2.31.0 (API calls) +- pyyaml>=6.0.1 (configuration files) + +**Note**: ShadowParse works without CTF dependencies - features gracefully disable if not installed. + +## Security Considerations + +1. **No Arbitrary Code Execution**: Auto-solvers analyze data safely +2. **Validation**: Hex payload validation before parsing +3. **Size Limits**: Auto-solvers limited to reasonable data sizes +4. **Timeout Protection**: API calls have 10-second timeouts +5. **Config Security**: ctf_config.yaml excluded from git +6. **Confirmation Required**: Flag submission requires explicit confirmation by default + +## Testing + +### Test Results +✅ Syntax validation - All modules compile without errors +✅ Basic scan mode - Backward compatible, no CTF overhead +✅ CTF mode activation - Modules load correctly +✅ Auto-solver execution - Successfully decodes Base64, ROT, XOR +✅ Challenge categorization - Correctly identifies challenge types +✅ Hint generation - Produces relevant, progressive hints +✅ Output file creation - All CTF output files generated +✅ Config template creation - YAML template generated correctly +✅ Code review - All issues addressed +✅ Security scan - No vulnerabilities detected (CodeQL) + +## Success Metrics Achieved + +✅ Users can run ShadowParse in CTF mode with one flag (--ctf-mode) +✅ 7+ auto-solver tools integrated (ROT, XOR, Substitution, CyberChef chains, QR/Barcode, Shellcode, Disassembly) +✅ Challenge categorization with 6 categories and confidence scoring +✅ Hint system provides context-aware progressive hints +✅ Flag submission works with CTFd, HTB, and webhooks +✅ All features work in both basic and deep scan modes +✅ Documentation updated with comprehensive CTF-mode examples +✅ Backward compatible - existing usage unaffected + +## Code Quality + +- **Modular Design**: Each CTF feature in separate module +- **Graceful Degradation**: Works without optional dependencies +- **Error Handling**: Comprehensive try-catch blocks +- **Type Hints**: Full type annotations +- **Documentation**: Inline comments and docstrings +- **Security**: No arbitrary code execution, validation, timeouts +- **Testing**: Verified with real PCAP files + +## Future Enhancements (Not in Scope) + +Potential future additions that could enhance the CTF features: +- Integration with more CTF platforms (PicoCTF, Root-Me, etc.) +- Machine learning-based flag pattern detection +- Automated writeup generation +- Team collaboration features +- Real-time hint updates during CTF events +- Advanced steganography detection (LSB analysis) +- Encrypted traffic decryption with known keys + +## Conclusion + +ShadowParse now includes a comprehensive CTF automation toolkit that significantly accelerates CTF challenge solving while maintaining the tool's core functionality and ease of use. The implementation is production-ready, well-tested, and follows security best practices. diff --git a/README.md b/README.md index 66e53a2..8176153 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,18 @@ ## ShadowParse: The Luxe PCAP Forensics Engine 🕵️‍♂️🛡️ -**ShadowParse** is a high-performance PCAP analysis and forensics tool designed for security researchers and CTF players. It combines deep packet inspection (DPI) with the **DeepRead Integration**, an automated multi-layered decoding engine. +**ShadowParse** is a high-performance PCAP analysis and forensics tool designed for security researchers and CTF players. It combines deep packet inspection (DPI) with the **DeepRead Integration**, an automated multi-layered decoding engine, and now includes comprehensive **CTF-specific features** for automated challenge solving. ## ✨ Key Features - **Dual Scan Modes**: - `Basic Scan`: Rapid analysis for quick flag hunting and traffic overviews. - `Deep Scan`: Full TCP/UDP stream reconstruction and multi-depth cipher analysis. - **DeepRead Universal Decoder**: Automatically detects and decodes over 40+ encodings and ciphers (Base64, Caesar, Rot47, Morse, Tap Code, etc.). +- **CTF Mode**: Specialized features for CTF challenges including: + - **Auto-Solver Library**: Automated solving for ROT, XOR, substitution ciphers, QR codes, barcodes + - **Challenge Categorization**: Intelligent detection of challenge type (Crypto, Forensics, Stego, Web, Network, Reverse) + - **Progressive Hints**: Context-aware hints based on findings + - **Flag Submission**: Integration with CTFd, HackTheBox, and custom webhooks + - **Tool Recommendations**: Suggests relevant tools based on challenge category - **Automatic Forensics**: Extracts files from HTTP traffic and identifies high-entropy payloads (potential encrypted C2 traffic). - **Comprehensive Reporting**: Generates interactive Markdown reports, JSON data exports, and filtered PCAPs of suspicious traffic. @@ -30,16 +36,38 @@ pip install -r requirements.txt ```bash python shadowparse.py -f evidence.pcap --basic-scan - ``` ### Deep Scan (Thorough) ```bash python shadowparse.py -f evidence.pcap -o ctf_report_folder +``` +### CTF Mode with Auto-Solver + +```bash +python shadowparse.py -f evidence.pcap --ctf-mode --auto-solve --hints ``` +### CTF Mode with Flag Submission + +```bash +# First, create and configure ctf_config.yaml +python shadowparse.py -f evidence.pcap --ctf-mode --auto-solve --submit-flags --config ctf_config.yaml +``` + +### Command-Line Options + +- `-f, --file`: PCAP file to analyze (required) +- `-o, --output`: Output folder name (default: shadow_deepscan_report) +- `-b, --basic-scan`: Run in fast Basic Scan mode +- `--ctf-mode`: Enable CTF-specific features (categorization, hints, enhanced reporting) +- `--auto-solve`: Run auto-solver tools (ROT, XOR, substitution ciphers, QR codes, etc.) +- `--submit-flags`: Enable flag submission to CTF platforms (requires --config) +- `--hints`: Show progressive hints based on findings +- `--config`: Path to CTF configuration file (ctf_config.yaml) + ## 📊 Output ShadowParse generates a structured report folder containing: @@ -49,6 +77,57 @@ ShadowParse generates a structured report folder containing: * `extracted_files/`: Any files recovered from the network streams. * `suspicious_traffic.pcap`: A filtered PCAP containing only the "weird" or suspicious packets for further analysis in Wireshark. +### Additional CTF Mode Outputs + +When `--ctf-mode` is enabled, ShadowParse also generates: + +* `ctf_analysis.md`: CTF-specific analysis including challenge category, hints, and tool recommendations. +* `auto_solver_results.txt`: Results from automated solving attempts (when `--auto-solve` is used). +* `suggested_tools.txt`: List of recommended tools based on challenge category. +* `flag_submissions.log`: Log of flag submission attempts (when `--submit-flags` is used). + +## 🎯 CTF Configuration + +To use flag submission features, create a `ctf_config.yaml` file: + +```yaml +ctf_platform: + enabled: true + platform: "ctfd" # Options: "ctfd", "htb", "webhook" + url: "https://ctf.example.com" + api_token: "your-api-token-here" + auto_submit: false # Set to true to auto-submit without confirmation +``` + +### Supported Platforms + +- **CTFd**: Popular CTF platform (requires challenge ID) +- **HackTheBox**: Machine flag submission (user.txt and root.txt) +- **Webhook**: Generic webhook for custom integrations + +## 🔧 CTF Auto-Solver Features + +The auto-solver can automatically attempt to decode: + +- **ROT Ciphers**: All ROT1-25 variations with readability scoring +- **XOR Encryption**: Single and multi-byte XOR bruteforce +- **Substitution Ciphers**: Frequency analysis-based solving +- **CyberChef Chains**: Common encoding chains (Base64→Gunzip→Hex, etc.) +- **QR Codes & Barcodes**: Automatic detection and decoding from images +- **Shellcode**: Pattern detection and disassembly +- **Binary Analysis**: Basic disassembly of extracted executables + +## 💡 Challenge Categories + +ShadowParse can automatically categorize challenges into: + +- **Cryptography**: High entropy, encoded data, cipher patterns +- **Forensics**: File carving, hidden data, metadata analysis +- **Steganography**: Image/audio files, LSB patterns +- **Web Exploitation**: HTTP traffic, SQL injection, XSS patterns +- **Network Analysis**: DNS tunneling, covert channels, protocol abuse +- **Reverse Engineering**: Binary extraction, shellcode, obfuscated code + ## ⚖️ License This project is licensed under the MIT License. diff --git a/ctf_categorizer.py b/ctf_categorizer.py new file mode 100644 index 0000000..53676f9 --- /dev/null +++ b/ctf_categorizer.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +""" +Challenge Categorization System +Intelligently categorizes PCAP analysis into CTF challenge types +""" +from typing import Dict, List, Tuple, Any +from collections import Counter +import re + + +class ChallengeCategorizer: + """Categorize CTF challenges based on PCAP analysis""" + + CATEGORIES = [ + 'Cryptography', + 'Forensics', + 'Steganography', + 'Web Exploitation', + 'Network Analysis', + 'Reverse Engineering' + ] + + def __init__(self): + self.category_scores = {cat: 0.0 for cat in self.CATEGORIES} + + def analyze_and_categorize(self, analysis_data: Dict[str, Any]) -> Dict[str, Any]: + """Analyze data and return categorization results""" + self.category_scores = {cat: 0.0 for cat in self.CATEGORIES} + + # Cryptography indicators + self._score_cryptography(analysis_data) + + # Forensics indicators + self._score_forensics(analysis_data) + + # Steganography indicators + self._score_steganography(analysis_data) + + # Web Exploitation indicators + self._score_web_exploitation(analysis_data) + + # Network Analysis indicators + self._score_network_analysis(analysis_data) + + # Reverse Engineering indicators + self._score_reverse_engineering(analysis_data) + + # Normalize scores to 0-1 range + max_score = max(self.category_scores.values()) if any(self.category_scores.values()) else 1 + normalized_scores = { + cat: score / max_score if max_score > 0 else 0 + for cat, score in self.category_scores.items() + } + + # Get primary and secondary categories + sorted_categories = sorted( + normalized_scores.items(), + key=lambda x: x[1], + reverse=True + ) + + primary_category = sorted_categories[0] if sorted_categories else ('Unknown', 0) + secondary_category = sorted_categories[1] if len(sorted_categories) > 1 and sorted_categories[1][1] > 0.3 else None + + return { + 'primary_category': primary_category[0], + 'primary_confidence': primary_category[1], + 'secondary_category': secondary_category[0] if secondary_category else None, + 'secondary_confidence': secondary_category[1] if secondary_category else 0, + 'all_scores': normalized_scores + } + + def _score_cryptography(self, data: Dict[str, Any]): + """Score cryptography indicators""" + score = 0.0 + + # High entropy data + if data.get('high_entropy_count', 0) > 0: + score += 3.0 * min(data['high_entropy_count'] / 10, 1.0) + + # Encoded data patterns + if data.get('base64_patterns', 0) > 0: + score += 2.0 + + if data.get('hex_patterns', 0) > 0: + score += 1.5 + + # Cipher keywords + crypto_keywords = ['cipher', 'encrypt', 'decrypt', 'key', 'aes', 'rsa', 'xor', 'rot', 'crypto', 'hash'] + for keyword in crypto_keywords: + if data.get('keywords', {}).get(keyword, 0) > 0: + score += 0.5 + + # Multiple encoding layers + if data.get('encoding_layers', 0) > 2: + score += 2.0 + + self.category_scores['Cryptography'] = score + + def _score_forensics(self, data: Dict[str, Any]): + """Score forensics indicators""" + score = 0.0 + + # File extractions + if data.get('files_extracted', 0) > 0: + score += 3.0 * min(data['files_extracted'] / 5, 1.0) + + # Suspicious traffic patterns + if data.get('suspicious_packets', 0) > 0: + score += 2.0 + + # Multiple protocols + if data.get('protocols_count', 0) > 5: + score += 1.5 + + # Large payloads + if data.get('large_payloads', 0) > 0: + score += 1.0 + + # Forensics keywords + forensics_keywords = ['forensic', 'evidence', 'hidden', 'carve', 'extract', 'recover'] + for keyword in forensics_keywords: + if data.get('keywords', {}).get(keyword, 0) > 0: + score += 0.5 + + self.category_scores['Forensics'] = score + + def _score_steganography(self, data: Dict[str, Any]): + """Score steganography indicators""" + score = 0.0 + + # Image files + if data.get('image_files', 0) > 0: + score += 4.0 + + # Audio files + if data.get('audio_files', 0) > 0: + score += 3.0 + + # LSB patterns (if detected) + if data.get('lsb_patterns', False): + score += 3.0 + + # Stego keywords + stego_keywords = ['steg', 'hide', 'hidden', 'embed', 'watermark', 'lsb'] + for keyword in stego_keywords: + if data.get('keywords', {}).get(keyword, 0) > 0: + score += 1.0 + + # Small modifications to images + if data.get('similar_images', 0) > 1: + score += 2.0 + + self.category_scores['Steganography'] = score + + def _score_web_exploitation(self, data: Dict[str, Any]): + """Score web exploitation indicators""" + score = 0.0 + + # HTTP traffic + if data.get('http_requests', 0) > 0: + score += 3.0 + + # SQL injection patterns + sql_patterns = ['union', 'select', 'or 1=1', 'drop table', 'exec', '--', '/*', '*/'] + if data.get('sql_injection_detected', False): + score += 4.0 + + # XSS patterns + xss_patterns = ['