utils.py

import os
import shutil
import random
import seaborn as sns
import matplotlib.pyplot as plt
import string
import shutil
import julia
import contextlib
import pandas as pd
import subprocess
import re

def generate_random_string(n = 10):
    return ''.join(random.choices(string.ascii_lowercase + string.digits, k=n))

with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
        julia.install()

        from julia import Main

        julia.Julia(compiled_modules=False)  

        Main.eval("""
            redirect_stdout(devnull)
            import Pkg
            Pkg.activate(".")  # Activate the environment in the current directory
            Pkg.instantiate()  # Ensure all dependencies are installed
        """)

        Main.include("script.jl")

async def repository_exist(folder_name):
    if os.path.exists(folder_name):
        shutil.rmtree(folder_name)

    os.makedirs(folder_name)


async def generate_histogram(data):

    sns.set(style="whitegrid")


    plt.figure(figsize=(10, 6))
    sns.histplot(data, bins=40, kde=True, color="skyblue", edgecolor="#1f77b4",  stat="probability")
    plt.title("Revenues distribution", fontsize=18, weight='bold', color="navy")
    plt.xlabel("Revenue", fontsize=14)
    plt.ylabel("Probability", fontsize=14 )
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12)

    # Add a legend
    plt.legend(["KDE (Density Estimate)", "Histogram"], fontsize=12)

    os.makedirs("images", exist_ok=True)

    plot_filename = "images/"+generate_random_string(n = 10)+".png"  
    plt.savefig(plot_filename, bbox_inches="tight")
    plt.close()

    return plot_filename

CONFIG = r"""
    \documentclass{article}
    \usepackage{graphicx}
    \usepackage{geometry}
    \geometry{a4paper, margin=1in}
    \usepackage{booktabs}
    \usepackage{caption}
    \usepackage{subcaption}
    \title{Portfolio optimization}
    \author{Aaltoes Adviser}
    \usepackage{eurosym}
    \begin{document}
    \maketitle
    """

async def generate_project_portfolio_report(user_id, folder, M,  budget, analysis=False, advanced=False, U=None, threshold=None, tolerance_level=None, max_violation=None):
    latex_content = CONFIG
    counter_table = 1
    counter_figure = 1
    images = [] 
    needed_columns = ["project","profit","cost", "risk","dependence"]
        
    latex_content +=r"""This report is automatically generated by python script, any unformatting is sad, but inevitable."""

    for j, item in enumerate(os.listdir(folder)):
        path = os.path.join(folder, item)
        output = Main.portfolio_optimization(path, M, budget, analysis = analysis, advanced = advanced, U=U, threshold=threshold, tolerance_level=tolerance_level, max_violation=max_violation)
        latex_content += r"""\newpage\section*{Optimization of Porfolio """+ str(j+1) +r"""}"""
        
        df = pd.read_csv(path)
        df["dependence"] = df["dependence"].fillna(r"""-""")
        df = df[needed_columns]

        latex_content += r"""\par For the portfolio """+ str(j+1) +r""", we are given next input data (see Table """+str(counter_table)+r"""):"""
        counter_table+=1

        latex_table = r"""\begin{table}[ht]
                    \centering
                    \caption{Input data for Portfolio """ + str(j+1) + r"""}
                    """ + df.to_latex(index=False, column_format="ccccc", float_format="%.2f") + r"""
                    \end{table}""" # index=False to exclude the row numbers
        latex_content += latex_table
        if output:
            latex_content +=r"""\par Based on the provided data, the model managed to find next solution that satisfies all of the constraints of the problem (see Table """+str(counter_table)+r"""):"""
            counter_table+=1
            latex_table_solution = r"""\begin{table}[ht]
    \centering
    \caption{Advised projects to invest for Portfolio """ + str(j+1) + r"""}
    """ + df[output[0].astype(bool)].to_latex(index=False, column_format="ccccc", float_format="%.2f") + r"""
    \end{table}
    """
            latex_content += latex_table_solution
            expected_profit =output[1]
            plot_revenue = await generate_histogram(output[2])
            images.append(plot_revenue)
            if analysis == True:
                plot_test_revenue = await generate_histogram(output[7])
                images.append(plot_test_revenue)
                
                latex_content += r"""
        \begin{figure}[ht]
            \centering
            % First subfigure
            \begin{subfigure}[b]{0.45\textwidth}  % Adjust width as needed
                \centering
                \includegraphics[width=\linewidth]{"""+plot_revenue+r"""}  % Replace with your image path
                \caption{Distribution of revenues on a train set}
                \label{fig:subfig1}
            \end{subfigure}
            \hfill
            % Second subfigure
            \begin{subfigure}[b]{0.45\textwidth}  % Adjust width as needed
                \centering
                \includegraphics[width=\linewidth]{"""+plot_test_revenue+r"""}  % Replace with your image path
                \caption{Distribution of revenues on a test set}
                \label{fig:subfig2}
            \end{subfigure}
            
            % Main figure caption
            \caption{Distribution of revenues given provided solution}
            \label{fig:mainfigure}
        \end{figure}
        \par As you can see from the plot (see Figure """+str(counter_figure)+r"""), the revenues on the test approximately follow the train set (if there are enough of scenarios).
           """
            else:
                latex_content += r"""
            \begin{figure}[ht]
             \centering
             \includegraphics[width=0.7\linewidth]{"""+plot_revenue+r"""}  
            \caption{Distribution of revenues}
            \label{fig:subfig1}
            \end{figure}
"""

            counter_figure+=1
            latex_content += r"""With given solution, we managed to get expected revenue of """ + str(round(expected_profit, 2)) + r"""{\euro}."""

            if analysis:
                perfect_information = output[5]
                scenario_price = output[6]
                latex_content += r""". Expected value of the perfect information is """+ str(round(perfect_information,2))+r"""{\euro}  and price of stochastic solution is """+str(round(scenario_price,2))+r"""{\euro}. """

            if advanced:
                p = output[-1]
                latex_content += r"""\par Since you prefered to use an extended version of problem formulation, we ensured that solution produces revenues of more than """+ str(threshold) +r""" in """ + str((1-p)*100)+r"""\% of scenarios. """

            latex_content += r"""\par The portfolio uses """+str(round(output[3], 2))+r"""/"""+str(budget)+r"""{\euro} from budget."""
        else:
            latex_content +=r"""\par Sorry, no feasible solution is found. Make sure that U is not too small."""
            if advanced:
                latex_content +=r"""You can also try to relax your parameters for R, T and tolerance level."""
        
    latex_content += r"""\end{document}"""

    tex_filename = str(user_id) + "_report.tex"

    with open(tex_filename, "w") as tex_file:
        tex_file.write(latex_content)

    os.environ["PATH"] += os.pathsep + "/Library/TeX/texbin"

    try:
        subprocess.run(["pdflatex", tex_filename], check=True)
        print("PDF успешно создан.")
        pdf_file = str(user_id) + "_report.pdf"
    except subprocess.CalledProcessError:
        print("Ошибка при компиляции LaTeX в PDF.")
        pdf_file = False
        

    for ext in [".aux", ".log", ".out", ".tex"]:
        temp_file = tex_filename.replace(".tex", ext)
        if os.path.exists(temp_file):
            os.remove(temp_file)
            print(f"{temp_file} удалён.")
        else:
            print(f"{temp_file} не найден.")

    for f in images:
        os.remove(f)

    await repository_exist(folder)

    return pdf_file


def check_format(s):
    allowed_pattern = re.compile(r'^\{\d*(,\d+)*\}$|^$')
    if pd.isna(s):
        return True
    return bool(allowed_pattern.match(s.strip()))

def check_data_csv(df):
    output = ""
    required_columns = ['profit', 'risk', 'project', 'cost', 'dependence']
    if set(required_columns).issubset(df.columns):
        print("All required columns are present.")
        if df[required_columns[:-1]].isnull().values.any():
            output +="\n-There are missing values in the DataFrame."
        else:
            print("There are no missing values in the DataFrame.")
            try:
                df['cost'] = pd.to_numeric(df['cost'], errors='coerce')
                df['profit'] = pd.to_numeric(df['profit'], errors='coerce')
                df['risk'] = pd.to_numeric(df['risk'], errors='coerce')

                if df['risk'].between(0, 1).all():
                    print("All values in the 'risk' column are between 0 and 1.")
                else:
                    output +="\n-Some values in the 'risk' column are outside the range 0 to 1"

                df['is_allowed_format'] = df['dependence'].apply(check_format)
                if df['is_allowed_format'].all():
                    print("Everything is fine wilh dependence column.")
                else:
                    output +="\n-Some values in dependence columns are not according to the allowed format."
            except:
                output +="\n-Non-numeric values found in cost, profit, risk columns."
    else:
        output += "\n-Some required columns are missing."
    if output:
        return output
    else:
        return False
    


def process_file(folder, path):
    try:
        if path.endswith(".xlsx"):
                
            print(f"Processing file: {path}")
            excel_data = pd.read_excel(path, sheet_name=None)

            for sheet_name, data in excel_data.items():

                csv_file_path = os.path.join(folder, generate_random_string()+".csv")
                data.to_csv(csv_file_path, index=False)
                print(f"Saved sheet '{sheet_name}' to '{csv_file_path}'")
                df = pd.read_csv(csv_file_path)
                output = check_data_csv(df)

                if output:
                    os.remove(csv_file_path)
                    os.remove(path)
                    return "In some of your tabs in .excel file there are errors:" + output
                
            os.remove(path)
            return False
    except:
        os.remove(path)
        return "-File seems to be broken"
    try:

        if path.endswith(".csv"):
            new_path = os.path.join(folder, generate_random_string()+".csv")
            
            os.rename(path, new_path)
            df = pd.read_csv(new_path)
            output = check_data_csv(df)
            if output:
                os.remove(new_path)
                return output
            else:
                return False
    except:
        os.remove(new_path)
        return "-File seems to be broken"

def to_number(value):
    try:
        return int(value)
    except ValueError:
        return float(value)