Feat/first draft

.github/workflows/benchmark.yml

@@ -0,0 +1,142 @@
+name: Dataframe Benchmarking
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:  # Allow manual trigger
+  schedule:
+    # Run weekly on Sundays at 06:00 UTC
+    - cron: '0 6 * * 0'
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python with uv
+      uses: astral-sh/setup-uv@v3
+      with:
+        enable-cache: true
+        cache-dependency-glob: "requirements.in"
+
+    - name: Set up Python
+      run: uv python install
+
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y bc
+
+    - name: Run benchmark suite
+      run: |
+        # Make run.sh executable
+        chmod +x run.sh
+
+        # Run the benchmark suite
+        bash run.sh
+      env:
+        # Set environment variable to indicate GitHub Actions
+        GITHUB_ACTIONS: true
+
+    - name: Display Summary Statistics
+      run: |
+        echo "::group::Summary Statistics"
+        if [ -f "outputs/summary_statistics.md" ]; then
+          cat outputs/summary_statistics.md
+        else
+          echo "Summary statistics file not found"
+        fi
+        echo "::endgroup::"
+
+    - name: Prepare benchmark artifacts
+      run: |
+        # Create a comprehensive benchmark report
+        mkdir -p benchmark-results
+
+        # Copy all output files
+        cp -r outputs/* benchmark-results/ 2>/dev/null || true
+        cp -r artifacts/* benchmark-results/ 2>/dev/null || true
+
+        # Create a summary report
+        cat > benchmark-results/README.md << 'EOF'
+        # Dataframe Benchmarking Results
+
+        This directory contains the complete results from the dataframe benchmarking suite.
+
+        ## Files Description
+
+        ### Performance Results
+        - `comparison_table.csv` - Main performance comparison across frameworks
+        - `summary_statistics.csv` - Statistical summary of benchmark results
+        - `summary_statistics.md` - Markdown formatted summary statistics
+        - `speedup_comparison.csv` - Relative performance metrics (if available)
+        - `cache_effectiveness.csv` - Cache performance analysis (if available)
+
+        ### Data Verification
+        - `hash_comparison.csv` - Hash comparison for result verification
+        - `processed_raw_data.csv` - Raw benchmark data
+
+        ### Detailed Results
+        - `results/` - Individual operation results by framework
+
+        ### Profiling Data
+        - `pyinstrument/` - Detailed profiling reports (HTML format)
+
+        ### Environment Info
+        - `requirements_*.txt` - Python environment details and package versions
+
+        ## Frameworks Tested
+
+        - **pandas**: Traditional dataframe library
+        - **fireducks**: High-performance pandas-compatible library  
+        - **polars**: Fast dataframes with lazy evaluation
+
+        ## Benchmark Operations
+
+        The suite tests 23 different operations including:
+        - Basic filtering and aggregations
+        - Complex multi-table joins
+        - Window functions and rolling operations
+        - String and datetime operations
+        - Statistical computations
+        - Large data concatenation and sorting
+
+        Generated on: $(date -u '+%Y-%m-%d %H:%M:%S UTC')
+        Commit: $GITHUB_SHA
+        Workflow: $GITHUB_RUN_ID
+        EOF
+
+    - name: Upload benchmark results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-results-${{ github.run_id }}
+        path: benchmark-results/
+        retention-days: 30
+        compression-level: 6
+
+    - name: Upload profiling reports
+      uses: actions/upload-artifact@v4
+      with:
+        name: profiling-reports-${{ github.run_id }}
+        path: outputs/pyinstrument/
+        retention-days: 30
+        compression-level: 6
+      if: always()
+
+    - name: Summary
+      run: |
+        echo "::notice title=Benchmark Complete::Benchmark suite completed successfully! Check the artifacts for detailed results."
+
+        # Display summary statistics if available
+        if [ -f "outputs/summary_statistics.md" ]; then
+          echo "::group::Summary Statistics"
+          cat outputs/summary_statistics.md
+          echo "::endgroup::"
+        else
+          echo "Summary statistics file not found"
+        fi

.gitignore

@@ -1,3 +1,8 @@
+# Generated files:
+artifacts/
+data/
+outputs/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[codz]

00_tools.py

@@ -0,0 +1,77 @@
+import hashlib
+import os
+import time
+from datetime import datetime
+from typing import Any, Callable, Dict
+
+
+def time_operation(
+    operation_name: str,
+    df_lib: Any,
+    func: Callable[..., Any],
+    *args: Any,
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Time a function and return the result and execution info"""
+
+    framework = df_lib.__name__
+    if framework == "fireducks.pandas":
+        framework = "fireducks"
+
+    start_time = time.perf_counter()
+    result = func(*args, **kwargs)
+    # Force evaluation for lazy operations
+    if hasattr(result, "compute"):
+        result = result.compute()
+    elif hasattr(result, "values"):
+        _ = result.values  # Access values to force computation
+    elif isinstance(result, df_lib.DataFrame):
+        _ = len(result)  # Force evaluation by accessing length
+    end_time = time.perf_counter()
+    execution_time = end_time - start_time
+    success = True
+    error_msg = None
+
+    # Save result to parquet file and compute hash
+    result_hash = None
+    if result is None:
+        raise ValueError("Operation returned None, which is not allowed.")
+
+    # Create outputs/results directory if it doesn't exist
+    results_dir = "outputs/results"
+    os.makedirs(results_dir, exist_ok=True)
+
+    # Save result to parquet file
+    output_filename = f"{results_dir}/{operation_name}_{framework}.parquet"
+
+    if hasattr(result, "to_frame"):
+        result = result.to_frame(name=operation_name)
+
+    if hasattr(result, "to_parquet"):
+        result.to_parquet(output_filename, index=False)
+    elif hasattr(result, "write_parquet"):
+        result.write_parquet(output_filename)
+    else:
+        # Handle other types (scalars, arrays, etc.)
+        # by converting to DataFrame
+        if hasattr(result, "__iter__") and not isinstance(result, (str, bytes)):
+            # Iterable but not string
+            temp_df = df_lib.DataFrame({"result": list(result)})
+        else:
+            # Scalar value
+            temp_df = df_lib.DataFrame({"result": [result]})
+        temp_df.to_parquet(output_filename, index=False)
+
+    # Compute hash of the saved file for consistency verification
+    with open(output_filename, "rb") as f:
+        result_hash = hashlib.sha256(f.read()).hexdigest()
+
+    return {
+        "operation": operation_name,
+        "framework": framework,
+        "execution_time": execution_time,
+        "success": success,
+        "error": error_msg,
+        "timestamp": datetime.now(),
+        "result_hash": result_hash,
+    }