Address-Parser/address_csv_converter.py at main · Apoorv012/Address-Parser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import pandas as pd

CSV_FILE = "your_address_data.csv"  # Your new structured address dataset
LABEL_MAPPING = {
    "house number": "HOUSE_NUMBER",
    "road": "ROAD",
    "sublocality": "SUBLOCALITY",
    "locality": "LOCALITY",
    "city": "CITY",
    "sub_district": "SUB_DISTRICT",
    "district": "DISTRICT",
    "state": "STATE",
    "poi": "POI",
    "pincode": "PINCODE"
}

def build_ner_training_data(csv_file):
    df = pd.read_csv(csv_file, dtype=str, sep="|").fillna("")  # Fill empty fields with empty string
    ner_data = []

    for _, row in df.iterrows():
        parts = []
        entities = []
        cursor = 0

        for col in [
            "house number", "road", "sublocality", "locality",
            "sub_district", "city", "district", "state", "poi", "pincode"
        ]:
            val = row[col].strip()
            if val:
                if parts:  # Add separator if not first
                    parts.append(", ")
                    cursor += 2

                start = cursor
                parts.append(val)
                cursor += len(val)
                end = cursor

                label = LABEL_MAPPING.get(col)
                if label:
                    entities.append((start, end, label))

        full_text = ''.join(parts)
        if full_text:
            ner_data.append((full_text, {"entities": entities}))

    return ner_data