From 4544ae8af841e1b61c5f18598f2a2a2667202b31 Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Mon, 18 Nov 2024 01:48:12 -0500 Subject: [PATCH 1/2] initial projection --- ml4opf/layers/projection.jl | 101 ++++++++++++++++++++++++++++++++++++ ml4opf/layers/projection.py | 56 ++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 ml4opf/layers/projection.jl create mode 100644 ml4opf/layers/projection.py diff --git a/ml4opf/layers/projection.jl b/ml4opf/layers/projection.jl new file mode 100644 index 0000000..3bf89be --- /dev/null +++ b/ml4opf/layers/projection.jl @@ -0,0 +1,101 @@ +using OPFGenerator, JuMP, MathOptInterface, PowerModels +using Ipopt, HiGHS, Clarabel, LinearAlgebra + +const PM = PowerModels + + +function projection( + OPF::Type{<:OPFGenerator.AbstractFormulation}, + data::OPFGenerator.OPFData, + solution, + optimizer +) where T + opf = OPFGenerator.build_opf(OPF, data, optimizer) + + obj = 0.0 + for v in vars_map[OPF] + if haskey(solution, v) + # obj += norm(solution[v] .- opf.model[v], p=2) + obj += sum((solution[v] .- opf.model[v]) .^ 2) + end + end + + JuMP.set_objective(opf.model, JuMP.MOI.MIN_SENSE, obj) + + OPFGenerator.solve!(opf) + + d = OPFGenerator.extract_result(opf) + + return d +end + + +config = nothing +base_data = nothing + + +vars_map = Dict{Type{<:OPFGenerator.AbstractFormulation}, Vector{Symbol}}( + OPFGenerator.ACOPF => [:pg, :qg, :pf, :pt, :qf, :qt, :vm, :va], + OPFGenerator.DCOPF => [:pg, :pf, :va], + OPFGenerator.SOCOPF => [:pg, :qg, :pf, :pt, :qf, :qt, :w, :wr, :wi], +) + + +function ac_projection(pd, qd, pg, qg, vm, va) + data = deepcopy(base_data) + data.pd .= pd + data.qd .= qd + return projection( + OPFGenerator.ACOPF, + data, + Dict( + :pg => pg, + :qg => qg, + :vm => vm, + :va => va + ), + Ipopt.Optimizer + ) +end + + +function dc_projection(pd, pg, pf, va) + data = deepcopy(base_data) + data.pd .= pd + return projection( + OPFGenerator.DCOPF, + data, + Dict( + :pg => pg, + :pf => pf, + :va => va + ), + HiGHS.Optimizer + ) +end + + +function soc_projection(pd, qd, pg, qg, w, wr, wi) + data = deepcopy(base_data) + data.pd .= pd + data.qd .= qd + return projection( + OPFGenerator.SOCOPF, + data, + Dict( + :pg => pg, + :qg => qg, + :w => w, + :wr => wr, + :wi => wi + ), + Clarabel.Optimizer + ) +end + + +function make_base_data(config) + # case_file = config |> OPFGenerator._get_case_name |> first # once OPFGenerator#141 is merged + case_file = joinpath(OPFGenerator.PGLib.PGLib_opf, OPFGenerator.PGLib.find_pglib_case(config["ref"])[1]) + return case_file |> PM.parse_file |> PM.make_basic_network |> OPFGenerator.OPFData +end diff --git a/ml4opf/layers/projection.py b/ml4opf/layers/projection.py new file mode 100644 index 0000000..9431c76 --- /dev/null +++ b/ml4opf/layers/projection.py @@ -0,0 +1,56 @@ +from uuid import uuid4 + +from ml4opf.formulations.problem import OPFProblem +from ml4opf.formulations.ac.problem import ACProblem +from ml4opf.formulations.dc.problem import DCProblem +from ml4opf.formulations.soc.problem import SOCProblem +from ml4opf import warn, __path__ as ml4opf_path + + +try: + from juliapkg import add, resolve +except ImportError as e: + raise ImportError("The projection layer depends on juliacall. Please install it by running \n\tpip install juliacall") from e + +# TODO: integrate with DiffOpt+ParametricOptInterface once POI#143 is merged +class Projection: + """Projection onto the feasible set. Implemented using juliacall and AI4OPT/OPFGenerator.""" + + initialized: bool = False + + def __init__(self, problem: OPFProblem): + self.problem = problem + self.setup() + + [warn("The projection layer is experimental! Proceed with caution.") for _ in range(3)] + + def setup(self): + from juliacall import Main + self.Module = Main.seval(f"module ML4OPF_Projection{uuid4()} end") + + if not Projection.initialized: + add("OPFGenerator", "5d2523b5-5e96-4b1c-8178-da2b93e9175f", url="https://github.com/AI4OPT/OPFGenerator") + add("MathOptInterface", "b8f27783-ece8-5eb3-8dc8-9495eed66fee") + add("JuMP", "4076af6c-e467-56ae-b986-b466b2749572") + add("PowerModels", "c36e90e8-916a-50a6-bd94-075b64ef4655") + add("Ipopt", "b6b21f68-93f8-5de0-b562-5493be1d77c9") + add("HiGHS", "87dc4568-4c63-4d18-b0c0-bb2238e4078b") + add("Clarabel", "61c947e1-3e6d-4ee4-985a-eec8c727bd6e") + resolve() + Projection.initialized = True + + self.Module.seval(f'include("{ml4opf_path[0]}/layers/projection.jl")') + + self.Module.config = self.problem.case_data["config"] + self.Module.seval("base_data = make_base_data(config)") + + if isinstance(self.problem, ACProblem): + self.project = self.Module.ac_projection + elif isinstance(self.problem, DCProblem): + self.project = self.Module.dc_projection + elif isinstance(self.problem, SOCProblem): + self.project = self.Module.soc_projection + + def __call__(self, *args): + return self.project(*args) + From 86cd673d2c9ef3626ba572f4bcfaf9b8e80cd7de Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Mon, 25 Nov 2024 01:28:52 -0500 Subject: [PATCH 2/2] rewrite with juliafunction --- ml4opf/layers/projection.jl | 34 ++++++------ ml4opf/layers/projection.py | 101 ++++++++++++++++++++++-------------- 2 files changed, 78 insertions(+), 57 deletions(-) diff --git a/ml4opf/layers/projection.jl b/ml4opf/layers/projection.jl index 3bf89be..e55ee74 100644 --- a/ml4opf/layers/projection.jl +++ b/ml4opf/layers/projection.jl @@ -1,7 +1,9 @@ -using OPFGenerator, JuMP, MathOptInterface, PowerModels -using Ipopt, HiGHS, Clarabel, LinearAlgebra +using OPFGenerator +using Ipopt, HiGHS, Clarabel -const PM = PowerModels +const PM = OPFGenerator.PowerModels +const JuMP = OPFGenerator.JuMP +const MOI = OPFGenerator.JuMP.MOI function projection( @@ -9,32 +11,28 @@ function projection( data::OPFGenerator.OPFData, solution, optimizer -) where T +) opf = OPFGenerator.build_opf(OPF, data, optimizer) obj = 0.0 - for v in vars_map[OPF] + for v in VARS_MAP[OPF] if haskey(solution, v) - # obj += norm(solution[v] .- opf.model[v], p=2) obj += sum((solution[v] .- opf.model[v]) .^ 2) end end - JuMP.set_objective(opf.model, JuMP.MOI.MIN_SENSE, obj) + JuMP.set_objective(opf.model, MOI.MIN_SENSE, obj) OPFGenerator.solve!(opf) d = OPFGenerator.extract_result(opf) - return d + return Tuple([ + d["primal"][string(v)] for v in VARS_MAP[OPF] + ]), nothing end - -config = nothing -base_data = nothing - - -vars_map = Dict{Type{<:OPFGenerator.AbstractFormulation}, Vector{Symbol}}( +VARS_MAP = Dict{Type{<:OPFGenerator.AbstractFormulation}, Vector{Symbol}}( OPFGenerator.ACOPF => [:pg, :qg, :pf, :pt, :qf, :qt, :vm, :va], OPFGenerator.DCOPF => [:pg, :pf, :va], OPFGenerator.SOCOPF => [:pg, :qg, :pf, :pt, :qf, :qt, :w, :wr, :wi], @@ -94,8 +92,6 @@ function soc_projection(pd, qd, pg, qg, w, wr, wi) end -function make_base_data(config) - # case_file = config |> OPFGenerator._get_case_name |> first # once OPFGenerator#141 is merged - case_file = joinpath(OPFGenerator.PGLib.PGLib_opf, OPFGenerator.PGLib.find_pglib_case(config["ref"])[1]) - return case_file |> PM.parse_file |> PM.make_basic_network |> OPFGenerator.OPFData -end +function no_backward(args...) + throw(ArgumentError("Projection layer does not support backpropagation.")) +end \ No newline at end of file diff --git a/ml4opf/layers/projection.py b/ml4opf/layers/projection.py index 9431c76..3b4f1b9 100644 --- a/ml4opf/layers/projection.py +++ b/ml4opf/layers/projection.py @@ -1,4 +1,16 @@ -from uuid import uuid4 +try: + from juliapkg import add, resolve + add("OPFGenerator", "5d2523b5-5e96-4b1c-8178-da2b93e9175f", url="https://github.com/AI4OPT/OPFGenerator") + add("Ipopt", "b6b21f68-93f8-5de0-b562-5493be1d77c9") + add("HiGHS", "87dc4568-4c63-4d18-b0c0-bb2238e4078b") + add("Clarabel", "61c947e1-3e6d-4ee4-985a-eec8c727bd6e") + add("DLPack", "53c2dc0f-f7d5-43fd-8906-6c0220547083") + add("Zygote", "e88e6eb3-aa80-5325-afca-941959d7151f") + resolve() + from juliafunction import JuliaFunction + from juliacall import Main +except ImportError as e: + raise ImportError("The projection layer depends on juliafunction. Please install it by running \n\tpip install git+https://github.com/klamike/juliafunction") from e from ml4opf.formulations.problem import OPFProblem from ml4opf.formulations.ac.problem import ACProblem @@ -7,50 +19,63 @@ from ml4opf import warn, __path__ as ml4opf_path -try: - from juliapkg import add, resolve -except ImportError as e: - raise ImportError("The projection layer depends on juliacall. Please install it by running \n\tpip install juliacall") from e - -# TODO: integrate with DiffOpt+ParametricOptInterface once POI#143 is merged -class Projection: - """Projection onto the feasible set. Implemented using juliacall and AI4OPT/OPFGenerator.""" +def make_projection_function(problem: OPFProblem): + """Create a projectionn layer for the given OPF problem. - initialized: bool = False + The following configurations are currently supported: - def __init__(self, problem: OPFProblem): - self.problem = problem - self.setup() + - ACProblem: input (pd, qd) with outputs (pg, qg, vm, va) - [warn("The projection layer is experimental! Proceed with caution.") for _ in range(3)] + - DCProblem: input (pd) with outputs (pg, pf, va) - def setup(self): - from juliacall import Main - self.Module = Main.seval(f"module ML4OPF_Projection{uuid4()} end") + - SOCProblem: input (pd, qd) with outputs (pg, qg, w, wr, wi) - if not Projection.initialized: - add("OPFGenerator", "5d2523b5-5e96-4b1c-8178-da2b93e9175f", url="https://github.com/AI4OPT/OPFGenerator") - add("MathOptInterface", "b8f27783-ece8-5eb3-8dc8-9495eed66fee") - add("JuMP", "4076af6c-e467-56ae-b986-b466b2749572") - add("PowerModels", "c36e90e8-916a-50a6-bd94-075b64ef4655") - add("Ipopt", "b6b21f68-93f8-5de0-b562-5493be1d77c9") - add("HiGHS", "87dc4568-4c63-4d18-b0c0-bb2238e4078b") - add("Clarabel", "61c947e1-3e6d-4ee4-985a-eec8c727bd6e") - resolve() - Projection.initialized = True - self.Module.seval(f'include("{ml4opf_path[0]}/layers/projection.jl")') + Note only one case can be loaded at a time (but different formulations for the same case are okay). - self.Module.config = self.problem.case_data["config"] - self.Module.seval("base_data = make_base_data(config)") + To parallelize batch samples using Julia threading, + set `PYTHON_JULIACALL_THREADS` and `PYTHON_JULIACALL_HANDLE_SIGNALS="yes"`. - if isinstance(self.problem, ACProblem): - self.project = self.Module.ac_projection - elif isinstance(self.problem, DCProblem): - self.project = self.Module.dc_projection - elif isinstance(self.problem, SOCProblem): - self.project = self.Module.soc_projection + Refer to the JuliaCall documentation on multi-threading in Julia for more details. + """ + warn( + """The projection layer is an experimental feature.\n""" + """The following configurations are currently supported: \n""" + """- ACProblem: input (pd, qd) with outputs (pg, qg, vm, va) \n""" + """- DCProblem: input (pd) with outputs (pg, pf, va) \n""" + """- SOCProblem: input (pd, qd) with outputs (pg, qg, w, wr, wi) \n""" + ) + Main.seval(f'using OPFGenerator; const PM = OPFGenerator.PowerModels') + Main.seval(f'config = nothing; base_data = nothing') - def __call__(self, *args): - return self.project(*args) + if "pglib_case" not in problem.case_data["config"] and "case_file" not in problem.case_data["config"]: + problem.case_data["config"]["pglib_case"] = problem.case_data["config"]["ref"] + Main.config = problem.case_data["config"] + Main.seval("base_data = config |> OPFGenerator._get_case_info |> first |> PM.parse_file |> PM.make_basic_network |> OPFGenerator.OPFData") + if isinstance(problem, ACProblem): + return JuliaFunction( + forward="ac_projection", + backward="no_backward", + batch_dims=[0] * 6, + include=f"{ml4opf_path[0]}/layers/projection.jl", + setup_code="base_data = Main.base_data", + ) + elif isinstance(problem, DCProblem): + return JuliaFunction( + forward="dc_projection", + backward="no_backward", + batch_dims=[0] * 4, + include=f"{ml4opf_path[0]}/layers/projection.jl", + setup_code="base_data = Main.base_data", + ) + elif isinstance(problem, SOCProblem): + return JuliaFunction( + forward="soc_projection", + backward="no_backward", + batch_dims=[0] * 7, + include=f"{ml4opf_path[0]}/layers/projection.jl", + setup_code="base_data = Main.base_data", + ) + else: + raise ValueError(f"Unsupported problem type: {type(problem)}") \ No newline at end of file