distributionAnalysis/labor.py at main · jaskeat/distributionAnalysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import marimo

__generated_with = "0.14.16"
app = marimo.App(width="medium")


@app.cell
def _():
    import marimo as mo
    import pandas as pd
    return (pd,)


@app.cell
def _():
    # import requests
    # from bs4 import BeautifulSoup
    # import io
    # import os

    # def get_tables_with_requests(url, headers=None):
    #     """
    #     More controlled approach using requests and BeautifulSoup
    #     """
    #     if headers is None:
    #         headers = {
    #             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    #         }

    #     try:
    #         # Get the webpage content
    #         response = requests.get(url, headers=headers)
    #         response.raise_for_status()

    #         # Parse with pandas
    #         tables = pd.read_html(io.StringIO(response.text))
    #         return tables

    #     except Exception as e:
    #         print(f"Error: {e}")
    #         return []

    # def filter_tables_by_size(tables, min_rows=2, min_cols=2):
    #     """Filter tables by minimum size"""
    #     filtered = []
    #     for i, table in enumerate(tables):
    #         if table.shape[0] >= min_rows and table.shape[1] >= min_cols:
    #             filtered.append((i, table))
    #     return filtered

    # def save_tables_to_csv(large_tables, base_filename="bls_florida_table"):
    #     """Save each table to a separate CSV file"""
    #     saved_files = []

    #     # Create output directory if it doesn't exist
    #     output_dir = "bls_tables"
    #     os.makedirs(output_dir, exist_ok=True)

    #     for table_index, table in large_tables:
    #         # Create filename
    #         filename = f"{base_filename}_{table_index}.csv"
    #         filepath = os.path.join(output_dir, filename)

    #         try:
    #             # Save to CSV
    #             table.to_csv(filepath, index=False)
    #             saved_files.append(filepath)
    #             print(f"Saved Table {table_index} to: {filepath}")
    #             print(f"  Shape: {table.shape}")
    #             print(f"  Columns: {list(table.columns)}")
    #             print()

    #         except Exception as e:
    #             print(f"Error saving table {table_index}: {e}")

    #     return saved_files

    # # Example with filtering and saving
    # url = "https://www.bls.gov/regions/southeast/news-release/countyemploymentandwages_florida.htm"
    # tables = get_tables_with_requests(url)

    # if tables:
    #     print(f"Total tables found: {len(tables)}")

    #     # Filter tables with at least 5 rows and 3 columns
    #     large_tables = filter_tables_by_size(tables, min_rows=5, min_cols=3)

    #     print(f"Large tables: {len(large_tables)}")

    #     # Display table info
    #     for table_index, table in large_tables:
    #         print(f"\nTable {table_index}: {table.shape}")
    #         print(table.head())

    #     # Save all large tables to CSV files
    #     print("\n" + "="*50)
    #     print("SAVING TABLES TO CSV FILES")
    #     print("="*50)

    #     saved_files = save_tables_to_csv(large_tables)

    #     print(f"\nSummary: Saved {len(saved_files)} tables")
    #     for file in saved_files:
    #         print(f"  - {file}")
    return


@app.cell
def _(pd):
    df = pd.read_csv('./bls_tables/bls_florida_table_2.csv')
    df
    return


@app.cell
def _():
    return


if __name__ == "__main__":
    app.run()