parse_cutoffs_2025.py

"""
MHT CET 2025 Cutoff Parsing Pipeline
Reads all 3 CAP round PDFs from cutoff_pdfs/year2025/ and produces:
  - processed_cutoffs_2025.json
"""

import os
import re
import json
import time
import pdfplumber
from collections import defaultdict

HEADER_FOOTER_SUBSTRINGS = [
    "government of maharashtra",
    "state common entrance",
    "cut off list",
    "degree courses",
    "master of engineering",
    "legends:",
    "maharashtra state seats",
    "figures in bracket indicates",
    "indicates maharashtra state general merit no.",
    "under graduate technical",
    "admissions a.y.",
]

def clean_text(text):
    if not text:
        return ""
    return re.sub(r'\s+', ' ', text).strip()

def group_words_to_lines(words, y_tolerance=3):
    lines = defaultdict(list)
    for w in words:
        matched_y = None
        w_top = w['top']
        for y in lines.keys():
            if abs(y - w_top) <= y_tolerance:
                matched_y = y
                break
        if matched_y is not None:
            lines[matched_y].append(w)
        else:
            lines[w_top].append(w)
    sorted_lines = []
    for y in sorted(lines.keys()):
        line_words = sorted(lines[y], key=lambda x: x['x0'])
        sorted_lines.append(line_words)
    return sorted_lines

def is_header_footer_line(line_words):
    line_str = " ".join([w['text'] for w in line_words]).lower()
    for sub in HEADER_FOOTER_SUBSTRINGS:
        if sub in line_str:
            return True
    if len(line_words) == 1 and line_words[0]['text'].isdigit():
        y = line_words[0]['top']
        if y > 750 or y < 50:
            return True
    return False

def is_rank_line(line_words):
    has_integer = False
    has_percentile = False
    for w in line_words:
        txt = w['text']
        if '(' in txt or ')' in txt:
            has_percentile = True
        elif txt.isdigit():
            has_integer = True
    return has_integer and not has_percentile

def is_percentile_line(line_words):
    for w in line_words:
        if '(' in w['text'] or ')' in w['text']:
            return True
    return False

def is_category_header_line(line_words):
    texts = [w['text'] for w in line_words]
    if 'Stage' in texts:
        return True
    all_upper_caps = True
    has_category = False
    for w in line_words:
        txt = w['text']
        if txt.isupper() and re.match(r'^[A-Z0-9\-]+$', txt):
            has_category = True
        else:
            if len(txt) == 1 and txt.isupper():
                continue
            all_upper_caps = False
            break
    return has_category and all_upper_caps

def cluster_headers(header_words):
    sorted_words = sorted(header_words, key=lambda w: w['x0'])
    clusters = []
    for w in sorted_words:
        if w['text'] == 'Stage':
            continue
        matched = False
        w_center = (w['x0'] + w['x1']) / 2
        for c in clusters:
            c_min_x = min([x['x0'] for x in c])
            c_max_x = max([x['x1'] for x in c])
            c_center = (c_min_x + c_max_x) / 2
            if not (w['x1'] < c_min_x or w['x0'] > c_max_x) or abs(w_center - c_center) <= 8:
                c.append(w)
                matched = True
                break
        if not matched:
            clusters.append([w])
    headers = []
    for c in clusters:
        c_sorted = sorted(c, key=lambda w: w['top'])
        name = "".join([w['text'] for w in c_sorted])
        x0 = min([w['x0'] for w in c_sorted])
        x1 = max([w['x1'] for w in c_sorted])
        center = (x0 + x1) / 2
        headers.append({'text': name, 'x0': x0, 'x1': x1, 'center': center})
    return sorted(headers, key=lambda h: h['x0'])

def print_progress(current, total, prefix="Processing", suffix="", bar_length=45):
    percent = float(current) * 100 / total
    filled_len = int(percent / 100 * bar_length)
    arrow = '=' * max(0, filled_len - 1) + ('>' if filled_len > 0 else '')
    spaces = ' ' * (bar_length - len(arrow))
    print(f"\r{prefix}: [{arrow}{spaces}] {percent:.1f}% ({current}/{total}) {suffix}", end='', flush=True)
    if current == total:
        print()

def parse_pdf(pdf_path, round_num):
    if not os.path.exists(pdf_path):
        print(f"\nWarning: File not found: {pdf_path}")
        return []

    records = []
    inst_code = inst_name = choice_code = course_name = None
    status = home_uni = seat_type = headers = None
    active_stages = []
    row_index = 0
    pending_rank_row = None

    def emit_pending():
        nonlocal pending_rank_row
        if pending_rank_row and headers:
            stage = pending_rank_row['stage_label']
            for w in pending_rank_row['words']:
                w_center = (w['x0'] + w['x1']) / 2
                closest_h = min(headers, key=lambda h: abs(h['center'] - w_center))
                dist = abs(closest_h['center'] - w_center)
                if dist <= 30:
                    rank_str = re.sub(r'\D', '', w['text'])
                    if rank_str:
                        records.append({
                            "round": round_num,
                            "institute_code": inst_code,
                            "institute_name": inst_name,
                            "choice_code": choice_code,
                            "course_name": course_name,
                            "status": status,
                            "home_university": home_uni,
                            "seat_type": seat_type,
                            "category": closest_h['text'],
                            "stage": stage,
                            "rank": int(rank_str),
                            "percentile": None
                        })
            pending_rank_row = None

    try:
        with pdfplumber.open(pdf_path) as pdf:
            total_pages = len(pdf.pages)
            for page_idx, page in enumerate(pdf.pages):
                suffix_info = f"Round {round_num} | Page {page_idx+1}/{total_pages}"
                print_progress(page_idx + 1, total_pages, prefix="Parsing PDF", suffix=suffix_info)

                words = page.extract_words()
                if not words:
                    continue

                lines = group_words_to_lines(words)
                content_lines = [line for line in lines if not is_header_footer_line(line)]

                line_idx = 0
                while line_idx < len(content_lines):
                    line = content_lines[line_idx]
                    line_str = " ".join([w['text'] for w in line])

                    inst_match = re.match(r"^(\d{5})\s*-\s*(.*)$", line_str)
                    if inst_match:
                        emit_pending()
                        inst_code = inst_match.group(1)
                        inst_name = clean_text(inst_match.group(2))
                        choice_code = course_name = status = home_uni = seat_type = headers = None
                        active_stages = []
                        row_index = 0
                        line_idx += 1
                        continue

                    choice_match = re.match(r"^(\d{10})\s*-\s*(.*)$", line_str)
                    if choice_match:
                        emit_pending()
                        choice_code = choice_match.group(1)
                        course_name = clean_text(choice_match.group(2))
                        status = home_uni = seat_type = headers = None
                        active_stages = []
                        row_index = 0
                        line_idx += 1
                        continue

                    if line_str.startswith("Status:"):
                        emit_pending()
                        status_match = re.match(r"^Status:\s*(.*?)\s*Home University\s*:\s*(.*?)$", line_str)
                        if status_match:
                            status = clean_text(status_match.group(1))
                            home_uni = clean_text(status_match.group(2))
                        else:
                            status = clean_text(line_str.replace("Status:", ""))
                            home_uni = None
                        seat_type = headers = None
                        active_stages = []
                        row_index = 0
                        line_idx += 1
                        continue

                    if is_category_header_line(line):
                        emit_pending()
                        header_words = []
                        while line_idx < len(content_lines) and not is_rank_line(content_lines[line_idx]) and not is_percentile_line(content_lines[line_idx]):
                            curr_str = " ".join([w['text'] for w in content_lines[line_idx]])
                            if re.match(r"^(\d{5})\s*-", curr_str) or re.match(r"^(\d{10})\s*-", curr_str) or curr_str.startswith("Status:"):
                                break
                            header_words.extend(content_lines[line_idx])
                            line_idx += 1
                        headers = cluster_headers(header_words)
                        row_index = 0
                        continue

                    if headers and is_rank_line(line):
                        emit_pending()
                        label_words = []
                        rank_words = []
                        for w in line:
                            if w['text'].isdigit():
                                rank_words.append(w)
                            else:
                                label_words.append(w)
                        stage_label = " ".join([w['text'] for w in label_words]).strip()
                        if stage_label:
                            if stage_label not in active_stages:
                                active_stages.append(stage_label)
                        else:
                            if row_index < len(active_stages):
                                stage_label = active_stages[row_index]
                            else:
                                stage_label = "I"
                        pending_rank_row = {'stage_label': stage_label, 'words': rank_words}
                        line_idx += 1
                        continue

                    if headers and is_percentile_line(line):
                        label_words = []
                        perc_words = []
                        for w in line:
                            if '(' in w['text'] or ')' in w['text']:
                                perc_words.append(w)
                            else:
                                label_words.append(w)
                        perc_label = " ".join([w['text'] for w in label_words]).strip()
                        stage = perc_label
                        if pending_rank_row:
                            stage = clean_text(pending_rank_row['stage_label'] + (" " + perc_label if perc_label else ""))
                        else:
                            stage = active_stages[row_index] if row_index < len(active_stages) else "I"

                        perc_matches = {}
                        for w in perc_words:
                            w_center = (w['x0'] + w['x1']) / 2
                            closest_h = min(headers, key=lambda h: abs(h['center'] - w_center))
                            if abs(closest_h['center'] - w_center) <= 30:
                                try:
                                    perc_matches[closest_h['text']] = float(w['text'].replace('(', '').replace(')', ''))
                                except ValueError:
                                    pass

                        rank_matches = {}
                        if pending_rank_row:
                            for w in pending_rank_row['words']:
                                w_center = (w['x0'] + w['x1']) / 2
                                closest_h = min(headers, key=lambda h: abs(h['center'] - w_center))
                                if abs(closest_h['center'] - w_center) <= 30:
                                    rank_matches[closest_h['text']] = int(w['text'])
                            pending_rank_row = None

                        for cat in set(list(rank_matches.keys()) + list(perc_matches.keys())):
                            records.append({
                                "round": round_num,
                                "institute_code": inst_code,
                                "institute_name": inst_name,
                                "choice_code": choice_code,
                                "course_name": course_name,
                                "status": status,
                                "home_university": home_uni,
                                "seat_type": seat_type,
                                "category": cat,
                                "stage": stage,
                                "rank": rank_matches.get(cat),
                                "percentile": perc_matches.get(cat)
                            })
                        row_index += 1
                        line_idx += 1
                        continue

                    if line_str.strip() and not line_str.isupper() and not line_str.isdigit():
                        emit_pending()
                        seat_type = clean_text(line_str)
                        headers = None
                        active_stages = []
                        row_index = 0

                    line_idx += 1

                emit_pending()

    except Exception as e:
        print(f"\nError processing {pdf_path}: {e}")
        import traceback
        traceback.print_exc()

    return records


def main():
    start_time = time.time()

    pdf_files = [
        ('cutoff_pdfs/year2025/2025ENGG_CAP1_CutOff.pdf', 1),
        ('cutoff_pdfs/year2025/2025ENGG_CAP2_CutOff.pdf', 2),
        ('cutoff_pdfs/year2025/2025ENGG_CAP3_CutOff.pdf', 3),
    ]

    all_records = []
    print("MHT CET 2025 Cutoff Parsing Pipeline")
    print("=====================================")

    for path, round_num in pdf_files:
        if not os.path.exists(path):
            print(f"Skipping Round {round_num} -- File not found: {path}")
            continue
        print(f"\nStarting Round {round_num} ({path})...")
        round_start = time.time()
        round_records = parse_pdf(path, round_num)
        all_records.extend(round_records)
        print(f"Completed Round {round_num} in {time.time()-round_start:.1f}s  --> {len(round_records):,} records")
        print("-" * 55)

    output_file = 'processed_cutoffs_2025.json'
    print(f"\nWriting {len(all_records):,} records to {output_file}...")
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(all_records, f, indent=2, ensure_ascii=False)

    file_mb = os.path.getsize(output_file) / (1024 * 1024)
    print(f"Done! JSON written ({file_mb:.1f} MB). Total: {time.time()-start_time:.1f}s")

if __name__ == '__main__':
    main()